425 lines
18 KiB
Python
425 lines
18 KiB
Python
"""Trusted checkout + hash-bound offline runtime inputs -> directory bundle."""
|
|
|
|
import base64
|
|
import csv
|
|
import io
|
|
import os
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
from . import manifest, proxy_runtime
|
|
from .policy import DeliveryError, contract, digest, fields, read_json, relative_name, sha256
|
|
|
|
ENTRYPOINTS = {
|
|
"kf-delivery": "scripts.release",
|
|
"kf-api": (
|
|
"uvicorn kf_oj.api.app:create_app --factory --host 127.0.0.1 --port 8000 "
|
|
"--no-proxy-headers --no-access-log --limit-concurrency 32 --backlog 64"
|
|
),
|
|
"kf-outbox-publisher": "kf_oj.workers.outbox",
|
|
"kf-problem-import-worker": "kf_oj.workers.problem_import",
|
|
"kf-accounts": "kf_oj.modules.identity.cli",
|
|
"kf-problems": "kf_oj.modules.problems.cli",
|
|
"kf-artifacts": "kf_oj.infra.artifacts.cli",
|
|
"kf-migrate": "alembic -c backend/alembic.ini",
|
|
}
|
|
|
|
|
|
def command(argv: list[str], cwd: Path, log: Path, env: dict | None = None) -> None:
|
|
with log.open("ab") as output:
|
|
output.write(("$ " + " ".join(argv) + "\n").encode())
|
|
output.flush()
|
|
subprocess.run(
|
|
argv,
|
|
cwd=cwd,
|
|
env=env,
|
|
stdout=output,
|
|
stderr=subprocess.STDOUT,
|
|
check=True,
|
|
timeout=1200,
|
|
)
|
|
|
|
|
|
def copy(source: Path, target: Path) -> None:
|
|
info = source.stat()
|
|
if not stat.S_ISREG(info.st_mode) or info.st_size > manifest.MAX_FILE_BYTES:
|
|
raise DeliveryError("unsupported source payload file")
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
with source.open("rb") as incoming, target.open("xb") as outgoing:
|
|
shutil.copyfileobj(incoming, outgoing)
|
|
target.chmod(0o555 if info.st_mode & 0o111 else 0o444)
|
|
|
|
|
|
def tree(source: Path, target: Path) -> None:
|
|
if source.is_symlink() or not source.is_dir():
|
|
raise DeliveryError("ordinary source directory required")
|
|
for directory, directories, names in os.walk(source, followlinks=False):
|
|
directories[:] = sorted(name for name in directories if name not in {"__pycache__", ".git"})
|
|
for name in directories:
|
|
if (Path(directory) / name).is_symlink():
|
|
raise DeliveryError("source directory link refused")
|
|
for name in sorted(names):
|
|
path = Path(directory) / name
|
|
if path.is_symlink():
|
|
raise DeliveryError("source file link refused")
|
|
if name.endswith((".pyc", ".pyo")) or name in {"_virtualenv.py", "_virtualenv.pth"}:
|
|
continue
|
|
if name.endswith(".pth"):
|
|
raise DeliveryError("dependency startup path hook refused")
|
|
relative = path.relative_to(source)
|
|
relative_name(
|
|
(target.name + "/" + relative.as_posix())
|
|
if target.name in {"site", "frontend"}
|
|
else "backend/" + relative.as_posix()
|
|
)
|
|
copy(path, target / relative)
|
|
|
|
|
|
def tracked(repo: Path, source_prefix: str, payload: Path) -> None:
|
|
"""Only tracked checkout members; never scan operator environment files."""
|
|
names = subprocess.check_output(["git", "ls-files", "-z", source_prefix], cwd=repo)
|
|
for raw in names.split(b"\0"):
|
|
if not raw:
|
|
continue
|
|
name = raw.decode()
|
|
relative_name(name)
|
|
source = repo / name
|
|
if source.is_symlink():
|
|
raise DeliveryError("tracked payload link refused")
|
|
copy(source, payload / name)
|
|
|
|
|
|
def dependency_records(site: Path) -> None:
|
|
"""Bind only shipped wheel members; venv console scripts are not shipped.
|
|
|
|
uv's generated console-script hashes include the temporary interpreter path.
|
|
Removing their RECORD rows is necessary for relocatable, repeatable payloads.
|
|
Included members retain their original verified wheel hashes.
|
|
"""
|
|
for record in sorted(site.glob("*.dist-info/RECORD")):
|
|
rows = []
|
|
for row in csv.reader(io.StringIO(record.read_text())):
|
|
if len(row) != 3:
|
|
raise DeliveryError("invalid dependency RECORD")
|
|
name, checksum, size = row
|
|
if name.startswith("../../../bin/"):
|
|
continue
|
|
if name.startswith("../../../include/site/python3.13/"):
|
|
# Wheel development headers, like console scripts, are not
|
|
# needed in the read-only service runtime.
|
|
continue
|
|
if "__pycache__" in Path(name).parts or name.endswith((".pyc", ".pyo")):
|
|
continue
|
|
relative_name("site/" + name)
|
|
member = site / name
|
|
if not member.exists():
|
|
raise DeliveryError("dependency RECORD member absent")
|
|
if member == record and checksum == size == "":
|
|
rows.append(row)
|
|
continue
|
|
expected = "sha256=" + base64.urlsafe_b64encode(
|
|
bytes.fromhex(digest(member))
|
|
).decode().rstrip("=")
|
|
if checksum != expected or size != str(member.stat().st_size):
|
|
raise DeliveryError("dependency RECORD fingerprint differs")
|
|
rows.append(row)
|
|
output = io.StringIO(newline="")
|
|
csv.writer(output, lineterminator="\n").writerows(sorted(rows))
|
|
record.chmod(0o600)
|
|
record.write_text(output.getvalue())
|
|
record.chmod(0o444)
|
|
|
|
|
|
def curated(root: Path, records: dict, name: str, target: Path) -> None:
|
|
"""Resolve only links explicitly bound by the separate trusted inventory."""
|
|
current = root / name
|
|
for _ in range(9):
|
|
relative = current.relative_to(root).as_posix()
|
|
entry = records.get(relative)
|
|
if entry is None:
|
|
raise DeliveryError("runtime input absent from trusted inventory")
|
|
if "link" not in entry:
|
|
if (
|
|
current.is_symlink()
|
|
or current.stat().st_size != entry["bytes"]
|
|
or digest(current) != entry["sha256"]
|
|
):
|
|
raise DeliveryError("runtime input fingerprint differs")
|
|
copy(current, target)
|
|
return
|
|
if not current.is_symlink() or os.readlink(current) != entry["link"]:
|
|
raise DeliveryError("runtime input link differs")
|
|
if entry["link"].startswith("/"):
|
|
# Public Arch libraries may contain /usr/lib aliases. Interpret
|
|
# them within the fingerprinted guest tree, never the build host.
|
|
if not relative.startswith("services/usr/lib/") or not entry["link"].startswith(
|
|
"/usr/lib/"
|
|
):
|
|
raise DeliveryError("absolute runtime input link refused")
|
|
current = Path(os.path.normpath(root / "services" / entry["link"].lstrip("/")))
|
|
else:
|
|
current = Path(os.path.normpath(current.parent / entry["link"]))
|
|
if not current.is_relative_to(root):
|
|
raise DeliveryError("runtime input escapes trusted root")
|
|
raise DeliveryError("runtime input link depth limit")
|
|
|
|
|
|
def image(source: Path, expected: str, target: Path) -> dict:
|
|
sha256(expected)
|
|
if source.is_symlink() or digest(source / "manifest.json") != expected:
|
|
raise DeliveryError("runtime catalog fingerprint differs")
|
|
catalog = read_json(source / "manifest.json")
|
|
identity = {
|
|
name: catalog[name]
|
|
for name in ("toolchain_id", "image_kind", "image_revision", "runtime_version")
|
|
}
|
|
identity["image_digest"] = expected
|
|
if catalog.get("format_version") != 2 or catalog.get("architecture") != "x86_64_v3":
|
|
raise DeliveryError("unsupported image catalog")
|
|
records = catalog.get("files")
|
|
directories = catalog.get("directories")
|
|
if (
|
|
not isinstance(records, list)
|
|
or not 1 <= len(records) <= 10000
|
|
or not isinstance(directories, list)
|
|
or len(directories) > 10000
|
|
):
|
|
raise DeliveryError("image catalog bounds")
|
|
target.mkdir(parents=True)
|
|
for name in directories:
|
|
relative_name("images/catalog/" + name + "/.__directory__")
|
|
original = source / name
|
|
if original.is_symlink() or not original.is_dir():
|
|
raise DeliveryError("runtime directory differs from catalog")
|
|
(target / name).mkdir(parents=True, exist_ok=True)
|
|
for item in records:
|
|
name = item["path"]
|
|
relative_name("images/catalog/" + name)
|
|
original = source / name
|
|
if (
|
|
original.is_symlink()
|
|
or original.stat().st_size != item["bytes"]
|
|
or digest(original) != item["sha256"]
|
|
):
|
|
raise DeliveryError("runtime file differs from catalog")
|
|
copy(original, target / name)
|
|
copy(source / "manifest.json", target / "manifest.json")
|
|
return identity
|
|
|
|
|
|
def build(repo: Path, inputs: Path, output: Path, work: Path) -> dict:
|
|
repo = repo.resolve(strict=True)
|
|
inputs = inputs.resolve(strict=True)
|
|
output = output.resolve()
|
|
work = work.resolve()
|
|
if output == work or output.is_relative_to(work) or work.is_relative_to(output):
|
|
raise DeliveryError("build output and work must be separate")
|
|
if subprocess.check_output(["git", "status", "--porcelain"], cwd=repo):
|
|
raise DeliveryError("build requires a clean committed checkout")
|
|
source_sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip()
|
|
contract(repo / "deploy/release/contract.v1.json")
|
|
fields_spec = {
|
|
"schema_version",
|
|
"runtime_root",
|
|
"runtime_manifest",
|
|
"runtime_manifest_sha256",
|
|
"images",
|
|
}
|
|
spec = read_json(inputs, 16384)
|
|
if (
|
|
not isinstance(spec, dict)
|
|
or type(spec.get("schema_version")) is not int
|
|
or spec["schema_version"] not in (1, 2)
|
|
):
|
|
raise DeliveryError("unsupported build inputs")
|
|
if spec["schema_version"] == 2:
|
|
fields_spec.add("proxy")
|
|
fields(spec, fields_spec, "build inputs")
|
|
if spec["schema_version"] == 2:
|
|
fields(spec["proxy"], {"root", "catalog_sha256"}, "proxy build input")
|
|
proxy_runtime.verify(Path(spec["proxy"]["root"]), spec["proxy"]["catalog_sha256"])
|
|
root = Path(spec["runtime_root"]).resolve(strict=True)
|
|
trusted_manifest = Path(spec["runtime_manifest"])
|
|
if digest(trusted_manifest) != sha256(spec["runtime_manifest_sha256"]):
|
|
raise DeliveryError("runtime inventory fingerprint differs")
|
|
runtime_records = read_json(trusted_manifest).get("files")
|
|
if not isinstance(runtime_records, list) or len(runtime_records) > 50000:
|
|
raise DeliveryError("runtime inventory bounds")
|
|
records = {entry["path"]: entry for entry in runtime_records}
|
|
if len(records) != len(runtime_records):
|
|
raise DeliveryError("duplicate runtime inventory member")
|
|
fields(spec["images"], {"gnu", "python313", "java21"}, "build images")
|
|
if output.exists() or output.is_symlink() or work.exists() or work.is_symlink():
|
|
raise DeliveryError("build output/work directories must be new")
|
|
if shutil.disk_usage(output.parent).free < 8 * 1024**3:
|
|
raise DeliveryError("build disk reserve below 8 GiB")
|
|
output.mkdir(mode=0o700)
|
|
work.mkdir(mode=0o700)
|
|
payload = output / "payload"
|
|
payload.mkdir()
|
|
locks = {
|
|
name: digest(repo / name)
|
|
for name in (
|
|
"backend/uv.lock",
|
|
"judge/Cargo.lock",
|
|
"frontend/pnpm-lock.yaml",
|
|
"deploy/release/contract.v1.json",
|
|
)
|
|
}
|
|
log = work / "build.log"
|
|
env = os.environ.copy()
|
|
env["UV_PROJECT_ENVIRONMENT"] = str(work / "venv")
|
|
env["CARGO_BUILD_JOBS"] = "2"
|
|
for argv, cwd in [
|
|
(
|
|
[
|
|
"uv",
|
|
"sync",
|
|
"--locked",
|
|
"--no-dev",
|
|
"--no-install-project",
|
|
"--no-build",
|
|
"--offline",
|
|
"--no-python-downloads",
|
|
"--python",
|
|
"3.13.16",
|
|
],
|
|
repo / "backend",
|
|
),
|
|
(
|
|
["pnpm", "install", "--frozen-lockfile", "--offline", "--ignore-scripts"],
|
|
repo / "frontend",
|
|
),
|
|
(["pnpm", "build"], repo / "frontend"),
|
|
(
|
|
[
|
|
str(Path.home() / ".cargo/bin/cargo"),
|
|
"build",
|
|
"--release",
|
|
"--workspace",
|
|
"--locked",
|
|
"--offline",
|
|
],
|
|
repo / "judge",
|
|
),
|
|
]:
|
|
command(argv, cwd, log, env)
|
|
command([str(work / "venv/bin/python"), "--version"], repo, log)
|
|
command(["node", "--version"], repo / "frontend", log)
|
|
command(["pnpm", "--version"], repo / "frontend", log)
|
|
command([str(Path.home() / ".cargo/bin/rustc"), "--version"], repo / "judge", log)
|
|
actual = {
|
|
"python": subprocess.check_output([str(work / "venv/bin/python"), "--version"], text=True)
|
|
.strip()
|
|
.removeprefix("Python "),
|
|
"node": subprocess.check_output(["node", "--version"], cwd=repo / "frontend", text=True)
|
|
.strip()
|
|
.removeprefix("v"),
|
|
"pnpm": subprocess.check_output(
|
|
["pnpm", "--version"], cwd=repo / "frontend", text=True
|
|
).strip(),
|
|
"rust": subprocess.check_output(
|
|
[str(Path.home() / ".cargo/bin/rustc"), "--version"], cwd=repo / "judge", text=True
|
|
).split()[1],
|
|
}
|
|
if actual != manifest.TOOLCHAINS:
|
|
raise DeliveryError("actual build toolchain differs from contract")
|
|
tracked(repo, "backend/src", payload)
|
|
tracked(repo, "backend/migrations", payload)
|
|
copy(repo / "backend/alembic.ini", payload / "backend/alembic.ini")
|
|
tree(work / "venv/lib/python3.13/site-packages", payload / "site")
|
|
dependency_records(payload / "site")
|
|
tree(repo / "frontend/dist", payload / "frontend")
|
|
tracked(repo, "deploy", payload)
|
|
for raw in subprocess.check_output(
|
|
["git", "ls-files", "-z", "scripts/release"], cwd=repo
|
|
).split(b"\0"):
|
|
if raw:
|
|
name = raw.decode()
|
|
relative_name("delivery/" + name)
|
|
copy(repo / name, payload / "delivery" / name)
|
|
for name in ("kf-sandbox", "kf-worker", "kf-coordinator", "kf-restore-io"):
|
|
copy(repo / "judge/target/release" / name, payload / "bin" / name)
|
|
for name in sorted(records):
|
|
if name.startswith("python/lib/") or name == "python/bin/python3.13":
|
|
if name.endswith((".pyc", ".pyo", ".a")) or any(
|
|
p
|
|
in {
|
|
"site-packages",
|
|
"__pycache__",
|
|
"test",
|
|
"tests",
|
|
"ensurepip",
|
|
"tkinter",
|
|
"idlelib",
|
|
}
|
|
for p in Path(name).parts
|
|
):
|
|
continue
|
|
relative_name(name)
|
|
curated(root, records, name, payload / name)
|
|
elif name.startswith("services/usr/lib/"):
|
|
relative = name.removeprefix("services/usr/lib/")
|
|
# Database-server OAuth/RADIUS plugins are provisioned with the
|
|
# database package, separate from application runtime libraries.
|
|
if relative.startswith(("libkrad.so", "libpq-oauth-")):
|
|
continue
|
|
if "/" not in relative and (".so" in relative or relative == "ld-linux-x86-64.so.2"):
|
|
relative_name("lib/" + relative)
|
|
curated(root, records, name, payload / "lib" / relative)
|
|
images = {}
|
|
for name, entry in spec["images"].items():
|
|
fields(entry, {"root", "catalog_sha256"}, "image build input")
|
|
images[name] = image(
|
|
Path(entry["root"]), entry["catalog_sha256"], payload / "images" / name
|
|
)
|
|
# The service inventory is an adjunct to the base GNU userland. Bind the
|
|
# latter's libc/loader/compiler runtime closure too, rather than relying on
|
|
# undeclared build-host libraries. The image was byte-verified above.
|
|
for library in sorted((payload / "images/gnu/usr/lib").iterdir()):
|
|
if not library.is_file() or ".so" not in library.name:
|
|
continue
|
|
target = payload / "lib" / library.name
|
|
if target.exists():
|
|
if digest(target) != digest(library):
|
|
raise DeliveryError("conflicting verified service library")
|
|
else:
|
|
copy(library, target)
|
|
for name, module in ENTRYPOINTS.items():
|
|
wrapper = (
|
|
"#!/bin/sh\nset -eu\n"
|
|
'APP_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)\n'
|
|
'export PYTHONHOME="$APP_ROOT/python"\n'
|
|
'export PYTHONPATH="$APP_ROOT/backend/src:$APP_ROOT/site:$APP_ROOT/delivery"\n'
|
|
"export PYTHONDONTWRITEBYTECODE=1\n"
|
|
'export LD_LIBRARY_PATH="$APP_ROOT/lib:$APP_ROOT/python/lib"\n'
|
|
'cd "$APP_ROOT"\n'
|
|
'exec "$APP_ROOT/python/bin/python3.13" -s -P -m ' + module + ' "$@"\n'
|
|
)
|
|
target = payload / "bin" / name
|
|
target.write_text(wrapper)
|
|
target.chmod(0o555)
|
|
if spec["schema_version"] == 2:
|
|
fields(spec["proxy"], {"root", "catalog_sha256"}, "proxy build input")
|
|
proxy_runtime.copy_to_payload(
|
|
Path(spec["proxy"]["root"]), spec["proxy"]["catalog_sha256"], payload / "proxy"
|
|
)
|
|
for directory, directories, _ in os.walk(payload, topdown=False):
|
|
for name in directories:
|
|
(Path(directory) / name).chmod(0o555)
|
|
payload.chmod(0o555)
|
|
if subprocess.check_output(["git", "status", "--porcelain"], cwd=repo):
|
|
raise DeliveryError("source changed during build")
|
|
data = manifest.write(output, source_sha, locks, images)
|
|
manifest.verify(output, digest(output / "manifest.json"))
|
|
return {
|
|
"source_sha": source_sha,
|
|
"manifest_sha256": digest(output / "manifest.json"),
|
|
"payload_bytes": data["payload_bytes"],
|
|
"files": len(data["files"]),
|
|
"installed": False,
|
|
"execution_verified": False,
|
|
}
|