From 6733d4e43523673fc5935c96672e28876ab7a8d0 Mon Sep 17 00:00:00 2001 From: kronecker <2026692687@qq.com> Date: Mon, 12 Oct 2026 15:15:00 +0800 Subject: [PATCH] feat(delivery): add bounded offline release tooling --- backend/tests/test_delivery.py | 92 +++++++++ deploy/release/README.md | 46 +++++ deploy/release/contract.v1.json | 27 +++ scripts/check_architecture.py | 15 +- scripts/release/README.md | 30 +++ scripts/release/__init__.py | 1 + scripts/release/__main__.py | 53 ++++++ scripts/release/build.py | 328 ++++++++++++++++++++++++++++++++ scripts/release/manifest.py | 256 +++++++++++++++++++++++++ scripts/release/policy.py | 161 ++++++++++++++++ 10 files changed, 1007 insertions(+), 2 deletions(-) create mode 100644 backend/tests/test_delivery.py create mode 100644 deploy/release/README.md create mode 100644 deploy/release/contract.v1.json create mode 100644 scripts/release/README.md create mode 100644 scripts/release/__init__.py create mode 100644 scripts/release/__main__.py create mode 100644 scripts/release/build.py create mode 100644 scripts/release/manifest.py create mode 100644 scripts/release/policy.py diff --git a/backend/tests/test_delivery.py b/backend/tests/test_delivery.py new file mode 100644 index 0000000..130f8fc --- /dev/null +++ b/backend/tests/test_delivery.py @@ -0,0 +1,92 @@ +"""Pure delivery-contract tests; these do not claim installation or VM safety.""" + +import importlib +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +policy = importlib.import_module("scripts.release.policy") +manifest = importlib.import_module("scripts.release.manifest") +architecture = importlib.import_module("scripts.check_architecture") + + +def test_delivery_authorities_and_layout(): + contract = policy.contract(ROOT / "deploy/release/contract.v1.json") + assert contract["services"]["worker"] == "none" + assert contract["services"]["sandbox-factory"] == "none" + assert contract["layout"]["task_cgroup"] != contract["layout"]["judge_state"] + assert contract["capacity"]["execution_slots"] == 1 + assert contract["recovery"]["consistent_cut_rpo_seconds"] == 0 + + +@pytest.mark.parametrize("value", [True, 0, 50001, "1"]) +def test_integer_resource_limits_cannot_be_coerced(value): + with pytest.raises(policy.DeliveryError): + policy.integer(value, 1, 50000, "entry count") + + +@pytest.mark.parametrize( + "name", + [ + "docs/a.md", + "backend/.env", + "backend/../secrets", + "/bin/tool", + "images/.ssh/id", + "site/entry.pyc", + ], +) +def test_private_and_noncanonical_paths_are_not_payload(name): + with pytest.raises(policy.DeliveryError): + policy.relative_name(name) + + +def test_expected_hash_is_external_and_not_placeholder(): + for value in ("0" * 64, "a" * 64, "bad"): + with pytest.raises(policy.DeliveryError): + policy.sha256(value) + + +def test_no_silent_contract_fields(tmp_path): + value = json.loads((ROOT / "deploy/release/contract.v1.json").read_text()) + value["ignore_isolation"] = True + path = tmp_path / "contract.json" + path.write_text(json.dumps(value)) + with pytest.raises(policy.DeliveryError, match="contract fields"): + policy.contract(path) + + +def test_duplicate_metadata_key_is_rejected(): + with pytest.raises(policy.DeliveryError, match="duplicate"): + json.loads( + '{"schema_version":1,"schema_version":2}', object_pairs_hook=policy.unique_object + ) + + +def test_inventory_of_trusted_read_only_files(tmp_path): + directory = tmp_path / "backend" + directory.mkdir() + file = directory / "version.txt" + file.write_bytes(b"accepted-source\n") + file.chmod(0o444) + directory.chmod(0o555) + entries = manifest.inventory(tmp_path) + assert entries[0]["path"] == "backend/version.txt" + assert entries[0]["bytes"] == 16 + assert manifest.directories(tmp_path) == ["backend"] + + +def test_delivery_cannot_import_database_state(tmp_path, monkeypatch): + (tmp_path / "bad.py").write_text("from kf_oj.modules.submissions.models import Evaluation\n") + monkeypatch.setattr(architecture, "RELEASE", tmp_path) + assert len(architecture.release_violations()) == 1 + + +def test_delivery_may_use_public_wire_contracts(tmp_path, monkeypatch): + (tmp_path / "public.py").write_text("from kf_oj.contracts.judging_v2 import NodeAdmission\n") + monkeypatch.setattr(architecture, "RELEASE", tmp_path) + assert architecture.release_violations() == [] diff --git a/deploy/release/README.md b/deploy/release/README.md new file mode 100644 index 0000000..e19bc1d --- /dev/null +++ b/deploy/release/README.md @@ -0,0 +1,46 @@ +# Single-node delivery contract + +`contract.v1.json` describes the first pre-release installation interface. It is +separate from judging v1/v2 wire formats and never rewrites historical reports. +The reference target is Arch/CachyOS Linux x86_64-v3, glibc 2.44, systemd and cgroup +v2. Images and service libraries are bound to explicit file/catalog fingerprints; +support for other distributions requires a separately verified delivery profile. + +The target has at least 4 GiB RAM and 12 GiB installation/data capacity. Judging +and problem-import concurrency remain one each. Configured service and execution +budgets, effective kernel limits and free capacity must also pass preflight; total +RAM alone does not prove enough capacity or verified execution isolation. + +Release payloads are directory bundles with a bounded manifest: at most 50,000 +single-link regular files, 256 MiB per file, 4 GiB total and 8 MiB metadata. The +manifest SHA-256 is supplied separately by the operator. Links, extra members, +private paths and unexpected file changes are refused. Built payloads and real +configuration never belong in Git. Validation is read-only and does not claim +that any service has been installed, started or verified: + +```sh +python3.13 -m scripts.release contract deploy/release/contract.v1.json +python3.13 -m scripts.release verify /path/to/bundle --manifest-sha256 SHA256_FROM_RECEIPT +``` + +Application versions live below `/opt/kf-onlinejudge/releases`; `current` selects +an explicitly installed version. Private operator/service configuration lives in +`/etc/kf-onlinejudge` and `/etc/kf-judge`, mutable application data under +`/var/lib/kf-onlinejudge`, and protected Sandbox state/images under +`/var/lib/kf-judge`. The delegated task cgroup is separate from the Root factory's +own supervision cgroup. Application processes do not initialize tables on start; +Backend Alembic remains the only migration entry point. + +Database login identities are separate for migration, API/outbox, Coordinator, +problem import and local identity operations. Rust Worker and Root factory do not +hold database credentials. Root factory has no MQ/platform/SSH/GitHub identity; +privileged execution remains inside Sandbox. Public and admin browser traffic +share the HTTPS site; execution-control interfaces remain internal. + +Upgrade and restore must retain immutable problem versions, submitted source, +audit and original judging reports. A consistent backup cut has a zero confirmed +data-loss target; restoring from already prepared release/backup materials has a +30-minute recovery target, including service health and actual four-language +verification. These are acceptance targets, not results inferred from this file. +Installation/service/restore evidence is required before this pre-release can be +treated as an accepted delivery; the contract itself does not deploy production. diff --git a/deploy/release/contract.v1.json b/deploy/release/contract.v1.json new file mode 100644 index 0000000..c578b79 --- /dev/null +++ b/deploy/release/contract.v1.json @@ -0,0 +1,27 @@ +{ + "schema_version": 1, + "format": "kf.single-node.directory.v1", + "platform": "linux_x86_64_v3_glibc244_systemd", + "toolchains": {"python": "3.13.16", "rust": "1.99.0", "node": "26.10.0", "pnpm": "11.28.4"}, + "capacity": {"memory_bytes": 4294967296, "disk_bytes": 12884901888, "execution_slots": 1, "import_slots": 1}, + "layout": { + "releases": "/opt/kf-onlinejudge/releases", + "current": "/opt/kf-onlinejudge/current", + "configuration": "/etc/kf-onlinejudge", + "data": "/var/lib/kf-onlinejudge", + "judge_configuration": "/etc/kf-judge", + "judge_state": "/var/lib/kf-judge", + "task_cgroup": "/sys/fs/cgroup/kf-judge-tasks" + }, + "services": { + "api": "platform", + "coordinator": "coordinator", + "outbox": "platform", + "problem-import": "problem_import", + "worker": "none", + "sandbox-factory": "none", + "identity-operator": "identity_operator", + "migration": "migration_owner" + }, + "recovery": {"consistent_cut_rpo_seconds": 0, "prepared_restore_rto_seconds": 1800} +} diff --git a/scripts/check_architecture.py b/scripts/check_architecture.py index b6dac3e..b9466cd 100644 --- a/scripts/check_architecture.py +++ b/scripts/check_architecture.py @@ -7,6 +7,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] PACKAGE = ROOT / "backend/src/kf_oj" +RELEASE = ROOT / "scripts/release" RUST_ALLOWED = { "kf-contracts": set(), "kf-sandbox": {"kf-contracts"}, @@ -97,12 +98,22 @@ def rust_violations() -> list[str]: return violations +def release_violations() -> list[str]: + """Offline delivery may use public contracts, never application state layers.""" + violations = [] + for path in sorted(RELEASE.glob("*.py")): + for target in imports(ast.parse(path.read_text(encoding="utf-8")), "scripts.release"): + if target.startswith("kf_oj.") and not target.startswith("kf_oj.contracts."): + violations.append(f"{path.name}: release cannot import application state: {target}") + return violations + + def main() -> None: - violations = python_violations() + rust_violations() + violations = python_violations() + rust_violations() + release_violations() if violations: print("\n".join(violations), file=sys.stderr) raise SystemExit(1) - print("Architecture boundaries: OK (Python imports + Rust crate graph)") + print("Architecture boundaries: OK (Python imports + Rust crate graph + offline delivery)") if __name__ == "__main__": diff --git a/scripts/release/README.md b/scripts/release/README.md new file mode 100644 index 0000000..efecab5 --- /dev/null +++ b/scripts/release/README.md @@ -0,0 +1,30 @@ +# Offline release tooling + +This package is an operator composition tool. `policy` validates delivery bounds, +`manifest` inventories/verifies explicit files, and `build` runs the locked trusted +checkout build and copies only tracked application/deployment sources, fresh +production dependencies and separately fingerprinted public runtime inputs. +It does not import domain ORM/repositories, write application state, implement a +second migration path, execute student code or perform Sandbox privilege work. +`scripts/check_architecture.py` enforces its application import boundary. + +All dependency resolution for a release uses existing lockfiles and offline +caches. Backend source is shipped with locked production dependencies rather +than invoking an unpinned isolated wheel build backend. uvx/pip and package +startup path hooks are not delivery runtime entry points. Build inputs specify a +verified public runtime inventory and the three existing image catalogs; an +independent expected SHA-256 binds each input. Real input paths/receipts are local +operator data and do not belong in Git. + +```sh +python3.13 -m scripts.release build --repo /native/checkout \ + --inputs /private/build-inputs.json --output /private/new-bundle \ + --work /private/new-build-work +``` + +The checkout must be clean and committed, build output/work directories new, +and the output filesystem have the required reserve. The resulting receipt +contains the source and manifest fingerprints, file count and byte total, with +`installed=false` and `execution_verified=false`. Installation and service +acceptance remain separate operations. See `deploy/release/README.md` for the +versioned installation contract and authority split. diff --git a/scripts/release/__init__.py b/scripts/release/__init__.py new file mode 100644 index 0000000..18c8257 --- /dev/null +++ b/scripts/release/__init__.py @@ -0,0 +1 @@ +"""Offline delivery tools; no application state or sandbox privilege operations.""" diff --git a/scripts/release/__main__.py b/scripts/release/__main__.py new file mode 100644 index 0000000..dda9672 --- /dev/null +++ b/scripts/release/__main__.py @@ -0,0 +1,53 @@ +"""Operator entry points; validation does not imply installed service health.""" + +import argparse +import json +import subprocess +from pathlib import Path + +from . import build, manifest, policy + + +def main() -> None: + parser = argparse.ArgumentParser(description="KF bounded offline delivery") + sub = parser.add_subparsers(dest="command", required=True) + check = sub.add_parser("contract") + check.add_argument("path", type=Path) + check = sub.add_parser("verify") + check.add_argument("bundle", type=Path) + check.add_argument("--manifest-sha256", required=True) + check = sub.add_parser("build") + check.add_argument("--repo", type=Path, required=True) + check.add_argument("--inputs", type=Path, required=True) + check.add_argument("--output", type=Path, required=True) + check.add_argument("--work", type=Path, required=True) + args = parser.parse_args() + try: + if args.command == "contract": + policy.contract(args.path) + result = {"contract_valid": True, "installed": False, "execution_verified": False} + elif args.command == "verify": + data = manifest.verify(args.bundle, args.manifest_sha256) + result = { + "bundle_valid": True, + "source_sha": data["source_sha"], + "payload_bytes": data["payload_bytes"], + "files": len(data["files"]), + "installed": False, + "execution_verified": False, + } + else: + result = build.build(args.repo, args.inputs, args.output, args.work) + print(json.dumps(result, sort_keys=True)) + except (policy.DeliveryError, OSError, subprocess.SubprocessError) as error: + # OSError filenames can include private operator paths; do not echo them. + parser.exit( + 1, + "Delivery refused: " + + (str(error) if isinstance(error, policy.DeliveryError) else type(error).__name__) + + "\n", + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/build.py b/scripts/release/build.py new file mode 100644 index 0000000..d4eca56 --- /dev/null +++ b/scripts/release/build.py @@ -0,0 +1,328 @@ +"""Trusted checkout + hash-bound offline runtime inputs -> directory bundle.""" + +import os +import shutil +import stat +import subprocess +from pathlib import Path + +from . import manifest +from .policy import DeliveryError, contract, digest, fields, read_json, relative_name, sha256 + +ENTRYPOINTS = { + "kf-api": ( + "uvicorn kf_oj.api.app:create_app --factory --host 127.0.0.1 --port 8000 " + "--no-proxy-headers --limit-concurrency 32 --backlog 64" + ), + "kf-outbox-publisher": "kf_oj.workers.outbox", + "kf-problem-import-worker": "kf_oj.workers.problem_import", + "kf-accounts": "kf_oj.modules.identity.cli", + "kf-problems": "kf_oj.modules.problems.cli", + "kf-artifacts": "kf_oj.infra.artifacts.cli", + "kf-migrate": "alembic -c backend/alembic.ini", +} + + +def command(argv: list[str], cwd: Path, log: Path, env: dict | None = None) -> None: + with log.open("ab") as output: + output.write(("$ " + " ".join(argv) + "\n").encode()) + output.flush() + subprocess.run( + argv, + cwd=cwd, + env=env, + stdout=output, + stderr=subprocess.STDOUT, + check=True, + timeout=1200, + ) + + +def copy(source: Path, target: Path) -> None: + info = source.stat() + if not stat.S_ISREG(info.st_mode) or info.st_size > manifest.MAX_FILE_BYTES: + raise DeliveryError("unsupported source payload file") + target.parent.mkdir(parents=True, exist_ok=True) + with source.open("rb") as incoming, target.open("xb") as outgoing: + shutil.copyfileobj(incoming, outgoing) + target.chmod(0o555 if info.st_mode & 0o111 else 0o444) + + +def tree(source: Path, target: Path) -> None: + if source.is_symlink() or not source.is_dir(): + raise DeliveryError("ordinary source directory required") + for directory, directories, names in os.walk(source, followlinks=False): + directories[:] = sorted(name for name in directories if name not in {"__pycache__", ".git"}) + for name in directories: + if (Path(directory) / name).is_symlink(): + raise DeliveryError("source directory link refused") + for name in sorted(names): + path = Path(directory) / name + if path.is_symlink(): + raise DeliveryError("source file link refused") + if name.endswith((".pyc", ".pyo")) or name in {"_virtualenv.py", "_virtualenv.pth"}: + continue + if name.endswith(".pth"): + raise DeliveryError("dependency startup path hook refused") + relative = path.relative_to(source) + relative_name( + (target.name + "/" + relative.as_posix()) + if target.name in {"site", "frontend"} + else "backend/" + relative.as_posix() + ) + copy(path, target / relative) + + +def tracked(repo: Path, source_prefix: str, payload: Path) -> None: + """Only tracked checkout members; never scan operator environment files.""" + names = subprocess.check_output(["git", "ls-files", "-z", source_prefix], cwd=repo) + for raw in names.split(b"\0"): + if not raw: + continue + name = raw.decode() + relative_name(name) + source = repo / name + if source.is_symlink(): + raise DeliveryError("tracked payload link refused") + copy(source, payload / name) + + +def curated(root: Path, records: dict, name: str, target: Path) -> None: + """Resolve only links explicitly bound by the separate trusted inventory.""" + current = root / name + for _ in range(9): + relative = current.relative_to(root).as_posix() + entry = records.get(relative) + if entry is None: + raise DeliveryError("runtime input absent from trusted inventory") + if "link" not in entry: + if ( + current.is_symlink() + or current.stat().st_size != entry["bytes"] + or digest(current) != entry["sha256"] + ): + raise DeliveryError("runtime input fingerprint differs") + copy(current, target) + return + if not current.is_symlink() or os.readlink(current) != entry["link"]: + raise DeliveryError("runtime input link differs") + if entry["link"].startswith("/"): + raise DeliveryError("absolute runtime input link refused") + current = Path(os.path.normpath(current.parent / entry["link"])) + if not current.is_relative_to(root): + raise DeliveryError("runtime input escapes trusted root") + raise DeliveryError("runtime input link depth limit") + + +def image(source: Path, expected: str, target: Path) -> dict: + sha256(expected) + if source.is_symlink() or digest(source / "manifest.json") != expected: + raise DeliveryError("runtime catalog fingerprint differs") + catalog = read_json(source / "manifest.json") + identity = { + name: catalog[name] + for name in ("toolchain_id", "image_kind", "image_revision", "runtime_version") + } + identity["image_digest"] = expected + if catalog.get("format_version") != 2 or catalog.get("architecture") != "x86_64_v3": + raise DeliveryError("unsupported image catalog") + records = catalog.get("files") + directories = catalog.get("directories") + if ( + not isinstance(records, list) + or not 1 <= len(records) <= 10000 + or not isinstance(directories, list) + or len(directories) > 10000 + ): + raise DeliveryError("image catalog bounds") + target.mkdir(parents=True) + for name in directories: + relative_name("images/catalog/" + name + "/.__directory__") + original = source / name + if original.is_symlink() or not original.is_dir(): + raise DeliveryError("runtime directory differs from catalog") + (target / name).mkdir(parents=True, exist_ok=True) + for item in records: + name = item["path"] + relative_name("images/catalog/" + name) + original = source / name + if ( + original.is_symlink() + or original.stat().st_size != item["bytes"] + or digest(original) != item["sha256"] + ): + raise DeliveryError("runtime file differs from catalog") + copy(original, target / name) + copy(source / "manifest.json", target / "manifest.json") + return identity + + +def build(repo: Path, inputs: Path, output: Path, work: Path) -> dict: + repo = repo.resolve(strict=True) + if subprocess.check_output(["git", "status", "--porcelain"], cwd=repo): + raise DeliveryError("build requires a clean committed checkout") + source_sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + contract(repo / "deploy/release/contract.v1.json") + fields_spec = { + "schema_version", + "runtime_root", + "runtime_manifest", + "runtime_manifest_sha256", + "images", + } + spec = fields(read_json(inputs, 16384), fields_spec, "build inputs") + if type(spec["schema_version"]) is not int or spec["schema_version"] != 1: + raise DeliveryError("unsupported build inputs") + root = Path(spec["runtime_root"]).resolve(strict=True) + trusted_manifest = Path(spec["runtime_manifest"]) + if digest(trusted_manifest) != sha256(spec["runtime_manifest_sha256"]): + raise DeliveryError("runtime inventory fingerprint differs") + runtime_records = read_json(trusted_manifest).get("files") + if not isinstance(runtime_records, list) or len(runtime_records) > 50000: + raise DeliveryError("runtime inventory bounds") + records = {entry["path"]: entry for entry in runtime_records} + if len(records) != len(runtime_records): + raise DeliveryError("duplicate runtime inventory member") + fields(spec["images"], {"gnu", "python313", "java21"}, "build images") + if output.exists() or output.is_symlink() or work.exists() or work.is_symlink(): + raise DeliveryError("build output/work directories must be new") + if shutil.disk_usage(output.parent).free < 8 * 1024**3: + raise DeliveryError("build disk reserve below 8 GiB") + output.mkdir(mode=0o700) + work.mkdir(mode=0o700) + payload = output / "payload" + payload.mkdir() + locks = { + name: digest(repo / name) + for name in ( + "backend/uv.lock", + "judge/Cargo.lock", + "frontend/pnpm-lock.yaml", + "deploy/release/contract.v1.json", + ) + } + log = work / "build.log" + env = os.environ.copy() + env["UV_PROJECT_ENVIRONMENT"] = str(work / "venv") + env["CARGO_BUILD_JOBS"] = "2" + for argv, cwd in [ + ( + [ + "uv", + "sync", + "--locked", + "--no-dev", + "--no-install-project", + "--no-build", + "--offline", + "--no-python-downloads", + "--python", + "3.13.16", + ], + repo / "backend", + ), + ( + ["pnpm", "install", "--frozen-lockfile", "--offline", "--ignore-scripts"], + repo / "frontend", + ), + (["pnpm", "build"], repo / "frontend"), + ( + [ + str(Path.home() / ".cargo/bin/cargo"), + "build", + "--release", + "--workspace", + "--locked", + "--offline", + ], + repo / "judge", + ), + ]: + command(argv, cwd, log, env) + command([str(work / "venv/bin/python"), "--version"], repo, log) + command(["node", "--version"], repo / "frontend", log) + command(["pnpm", "--version"], repo / "frontend", log) + command([str(Path.home() / ".cargo/bin/rustc"), "--version"], repo / "judge", log) + actual = { + "python": subprocess.check_output([str(work / "venv/bin/python"), "--version"], text=True) + .strip() + .removeprefix("Python "), + "node": subprocess.check_output(["node", "--version"], cwd=repo / "frontend", text=True) + .strip() + .removeprefix("v"), + "pnpm": subprocess.check_output( + ["pnpm", "--version"], cwd=repo / "frontend", text=True + ).strip(), + "rust": subprocess.check_output( + [str(Path.home() / ".cargo/bin/rustc"), "--version"], cwd=repo / "judge", text=True + ).split()[1], + } + if actual != manifest.TOOLCHAINS: + raise DeliveryError("actual build toolchain differs from contract") + tracked(repo, "backend/src", payload) + tracked(repo, "backend/migrations", payload) + copy(repo / "backend/alembic.ini", payload / "backend/alembic.ini") + tree(work / "venv/lib/python3.13/site-packages", payload / "site") + tree(repo / "frontend/dist", payload / "frontend") + tracked(repo, "deploy", payload) + for name in ("kf-sandbox", "kf-worker", "kf-coordinator"): + copy(repo / "judge/target/release" / name, payload / "bin" / name) + for name in sorted(records): + if name.startswith("python/lib/") or name == "python/bin/python3.13": + if name.endswith((".pyc", ".pyo", ".a")) or any( + p + in { + "site-packages", + "__pycache__", + "test", + "tests", + "ensurepip", + "tkinter", + "idlelib", + } + for p in Path(name).parts + ): + continue + relative_name(name) + curated(root, records, name, payload / name) + elif name.startswith("services/usr/lib/"): + relative = name.removeprefix("services/usr/lib/") + if "/" not in relative and (".so" in relative or relative == "ld-linux-x86-64.so.2"): + relative_name("lib/" + relative) + curated(root, records, name, payload / "lib" / relative) + images = {} + for name, entry in spec["images"].items(): + fields(entry, {"root", "catalog_sha256"}, "image build input") + images[name] = image( + Path(entry["root"]), entry["catalog_sha256"], payload / "images" / name + ) + for name, module in ENTRYPOINTS.items(): + wrapper = ( + "#!/bin/sh\nset -eu\n" + 'APP_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)\n' + 'export PYTHONHOME="$APP_ROOT/python"\n' + 'export PYTHONPATH="$APP_ROOT/backend/src:$APP_ROOT/site"\n' + "export PYTHONDONTWRITEBYTECODE=1\n" + 'export LD_LIBRARY_PATH="$APP_ROOT/lib:$APP_ROOT/python/lib"\n' + 'cd "$APP_ROOT"\n' + 'exec "$APP_ROOT/python/bin/python3.13" -s -P -m ' + module + ' "$@"\n' + ) + target = payload / "bin" / name + target.write_text(wrapper) + target.chmod(0o555) + for directory, directories, _ in os.walk(payload, topdown=False): + for name in directories: + (Path(directory) / name).chmod(0o555) + payload.chmod(0o555) + if subprocess.check_output(["git", "status", "--porcelain"], cwd=repo): + raise DeliveryError("source changed during build") + data = manifest.write(output, source_sha, locks, images) + manifest.verify(output, digest(output / "manifest.json")) + return { + "source_sha": source_sha, + "manifest_sha256": digest(output / "manifest.json"), + "payload_bytes": data["payload_bytes"], + "files": len(data["files"]), + "installed": False, + "execution_verified": False, + } diff --git a/scripts/release/manifest.py b/scripts/release/manifest.py new file mode 100644 index 0000000..89fdea0 --- /dev/null +++ b/scripts/release/manifest.py @@ -0,0 +1,256 @@ +"""Explicit directory bundle inventories, verified before installation.""" + +import os +import re +import stat +from pathlib import Path + +from .policy import ( + FORMAT, + MAX_BUNDLE_BYTES, + MAX_FILE_BYTES, + MAX_FILES, + MAX_METADATA_BYTES, + PLATFORM, + TOOLCHAINS, + DeliveryError, + canonical, + digest, + fields, + integer, + read_json, + relative_name, + sha256, +) + + +def inventory(root: Path) -> list[dict]: + """No links/devices or inferred inclusion; bounds apply before hashing.""" + if not root.is_dir() or root.is_symlink(): + raise DeliveryError("payload root must be an ordinary directory") + entries = [] + total = 0 + directory_count = 0 + for directory, directories, files in os.walk(root, followlinks=False): + directory_count += len(directories) + if directory_count + len(entries) + len(files) > MAX_FILES: + raise DeliveryError("payload entry count limit") + for name in directories: + path = Path(directory) / name + if path.is_symlink(): + raise DeliveryError("payload directory link refused") + for name in sorted(files): + path = Path(directory) / name + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise DeliveryError("payload must contain single-link regular files") + if stat.S_IMODE(info.st_mode) not in (0o444, 0o555): + raise DeliveryError("payload file must be read-only") + size = integer(info.st_size, 0, MAX_FILE_BYTES, "payload file size") + total += size + if total > MAX_BUNDLE_BYTES or len(entries) >= MAX_FILES: + raise DeliveryError("payload aggregate limit") + relative = relative_name(path.relative_to(root).as_posix()) + entries.append( + { + "path": relative, + "bytes": size, + "sha256": digest(path), + "mode": "0555" if info.st_mode & 0o111 else "0444", + } + ) + return sorted(entries, key=lambda x: x["path"]) + + +def directories(root: Path) -> list[str]: + result = [] + for directory, names, _ in os.walk(root, followlinks=False): + for name in names: + path = Path(directory) / name + value = path.relative_to(root).as_posix() + relative_name(value + "/.__directory__") + if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o555: + raise DeliveryError("payload directories must be read-only ordinary directories") + result.append(value) + if len(result) > MAX_FILES: + raise DeliveryError("payload directory count limit") + return sorted(result) + + +def validate(data: dict) -> dict: + fields( + data, + { + "schema_version", + "format", + "release_id", + "source_sha", + "platform", + "toolchains", + "locks", + "images", + "files", + "directories", + "payload_bytes", + }, + "bundle", + ) + if ( + type(data["schema_version"]) is not int + or data["schema_version"] != 1 + or data["format"] != FORMAT + ): + raise DeliveryError("unsupported bundle format") + if not isinstance(data["source_sha"], str) or not re.fullmatch( + r"[0-9a-f]{40}", data["source_sha"] + ): + raise DeliveryError("invalid source SHA") + if ( + not isinstance(data["release_id"], str) + or data["release_id"] != "sd-" + data["source_sha"][:12] + ): + raise DeliveryError("release identity mismatch") + if data["platform"] != PLATFORM or data["toolchains"] != TOOLCHAINS: + raise DeliveryError("bundle platform/toolchain mismatch") + fields( + data["locks"], + { + "backend/uv.lock", + "judge/Cargo.lock", + "frontend/pnpm-lock.yaml", + "deploy/release/contract.v1.json", + }, + "lock fingerprints", + ) + for value in data["locks"].values(): + sha256(value) + fields(data["images"], {"gnu", "python313", "java21"}, "runtime images") + for image in data["images"].values(): + fields( + image, + {"toolchain_id", "image_kind", "image_revision", "runtime_version", "image_digest"}, + "runtime identity", + ) + sha256(image["image_digest"]) + integer(image["image_revision"], 1, 2**32 - 1, "image revision") + for name in ("toolchain_id", "image_kind", "runtime_version"): + if not isinstance(image[name], str) or not re.fullmatch( + r"[A-Za-z0-9_.+-]{1,128}", image[name] + ): + raise DeliveryError("invalid runtime identity") + runtimes = { + "gnu": ("gnu-cxx-v1", "linux_gnu_cxx_x86_64_v1", None), + "python313": ("cpython313-v1", "linux_cpython313_x86_64_v1", "3.13.16"), + "java21": ("openjdk21-v1", "linux_openjdk21_x86_64_v1", "21.0.12.1+1-LTS"), + } + for name, (toolchain, kind, version) in runtimes.items(): + image = data["images"][name] + if ( + image["toolchain_id"] != toolchain + or image["image_kind"] != kind + or (version is not None and image["runtime_version"] != version) + ): + raise DeliveryError("unsupported runtime image") + files = data["files"] + if not isinstance(files, list) or not 1 <= len(files) <= MAX_FILES: + raise DeliveryError("bundle file count") + seen = set() + total = 0 + for entry in files: + fields(entry, {"path", "bytes", "sha256", "mode"}, "bundle file") + path = relative_name(entry["path"]) + if path in seen or entry["mode"] not in {"0444", "0555"}: + raise DeliveryError("duplicate file or unsafe mode") + seen.add(path) + total += integer(entry["bytes"], 0, MAX_FILE_BYTES, "bundle file size") + sha256(entry["sha256"]) + if ( + total > MAX_BUNDLE_BYTES + or type(data["payload_bytes"]) is not int + or data["payload_bytes"] != total + ): + raise DeliveryError("bundle byte total") + required = { + "bin/kf-sandbox", + "bin/kf-worker", + "bin/kf-coordinator", + "python/bin/python3.13", + "backend/alembic.ini", + "frontend/index.html", + "frontend/admin/index.html", + "deploy/release/contract.v1.json", + } + required.update(f"images/{name}/manifest.json" for name in data["images"]) + if not required <= seen: + raise DeliveryError("incomplete application bundle") + directory_names = data["directories"] + if not isinstance(directory_names, list) or len(directory_names) + len(files) > MAX_FILES: + raise DeliveryError("bundle directory count") + directory_set = set() + for name in directory_names: + if not isinstance(name, str): + raise DeliveryError("invalid bundle directory") + relative_name(name + "/.__directory__") + if name in directory_set or name in seen: + raise DeliveryError("duplicate bundle directory") + directory_set.add(name) + for name in seen | directory_set: + for parent in Path(name).parents: + if parent.as_posix() != "." and parent.as_posix() not in directory_set: + raise DeliveryError("missing payload parent directory") + return data + + +def verify(bundle: Path, expected_sha256: str) -> dict: + """The manifest digest comes from a separate trusted operator receipt.""" + expected_sha256 = sha256(expected_sha256) + if not bundle.is_dir() or bundle.is_symlink(): + raise DeliveryError("bundle directory required") + metadata = bundle / "manifest.json" + info = metadata.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_size > MAX_METADATA_BYTES: + raise DeliveryError("unsafe manifest file") + if digest(metadata) != expected_sha256: + raise DeliveryError("bundle manifest fingerprint mismatch") + if set(p.name for p in bundle.iterdir()) != {"manifest.json", "payload"}: + raise DeliveryError("unexpected bundle member") + data = validate(read_json(metadata)) + if inventory(bundle / "payload") != sorted(data["files"], key=lambda x: x["path"]): + raise DeliveryError("bundle contents differ from manifest") + if directories(bundle / "payload") != sorted(data["directories"]): + raise DeliveryError("bundle directories differ from manifest") + for name, identity in data["images"].items(): + if digest(bundle / "payload/images" / name / "manifest.json") != identity["image_digest"]: + raise DeliveryError("image catalog differs from runtime identity") + if ( + digest(bundle / "payload/deploy/release/contract.v1.json") + != data["locks"]["deploy/release/contract.v1.json"] + ): + raise DeliveryError("delivery contract fingerprint mismatch") + return data + + +def write(bundle: Path, source_sha: str, locks: dict, images: dict) -> dict: + files = inventory(bundle / "payload") + data = validate( + { + "schema_version": 1, + "format": FORMAT, + "release_id": "sd-" + source_sha[:12], + "source_sha": source_sha, + "platform": PLATFORM, + "toolchains": TOOLCHAINS, + "locks": locks, + "images": images, + "files": files, + "directories": directories(bundle / "payload"), + "payload_bytes": sum(x["bytes"] for x in files), + } + ) + raw = canonical(data) + if len(raw) > MAX_METADATA_BYTES: + raise DeliveryError("bundle manifest exceeds metadata limit") + with (bundle / "manifest.json").open("xb") as stream: + stream.write(raw) + (bundle / "manifest.json").chmod(0o444) + return data diff --git a/scripts/release/policy.py b/scripts/release/policy.py new file mode 100644 index 0000000..e352b5a --- /dev/null +++ b/scripts/release/policy.py @@ -0,0 +1,161 @@ +"""Versioned delivery bounds, separate from the judging wire protocols.""" + +import hashlib +import json +import re +from pathlib import Path +from typing import Any + +MAX_METADATA_BYTES = 8 * 1024 * 1024 +MAX_FILES = 50000 +MAX_FILE_BYTES = 256 * 1024 * 1024 +MAX_BUNDLE_BYTES = 4 * 1024**3 +FORMAT = "kf.single-node.directory.v1" +PLATFORM = "linux_x86_64_v3_glibc244_systemd" +TOOLCHAINS = {"python": "3.13.16", "rust": "1.99.0", "node": "26.10.0", "pnpm": "11.28.4"} +ROOTS = ("backend/", "python/", "site/", "lib/", "bin/", "frontend/", "images/", "deploy/") +FORBIDDEN = {".git", ".localplan", "docs", "__pycache__", ".env", ".ssh"} + + +class DeliveryError(ValueError): + """An intentionally bounded, non-secret error for the operator.""" + + +def digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def sha256(value: Any) -> str: + if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value): + raise DeliveryError("invalid SHA-256") + if value == "a" * 64 or value == "0" * 64: + raise DeliveryError("placeholder SHA-256 refused") + return value + + +def fields(value: Any, expected: set[str], label: str) -> dict: + if not isinstance(value, dict) or set(value) != expected: + raise DeliveryError(f"invalid {label} fields") + return value + + +def integer(value: Any, lower: int, upper: int, label: str) -> int: + if type(value) is not int or not lower <= value <= upper: + raise DeliveryError(f"invalid {label}") + return value + + +def relative_name(value: Any) -> str: + if not isinstance(value, str) or len(value) > 512: + raise DeliveryError("invalid payload path") + if not all(re.fullmatch(r"[A-Za-z0-9_.+@-]{1,255}", p) for p in value.split("/")): + raise DeliveryError("noncanonical payload path") + if any(p in {".", "..", *FORBIDDEN} for p in value.split("/")): + raise DeliveryError("private or unsafe payload path") + if not value.startswith(ROOTS) or value.endswith((".pyc", ".pyo")): + raise DeliveryError("payload path outside delivery allowlist") + return value + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise DeliveryError("duplicate JSON key") + result[key] = value + return result + + +def read_json(path: Path, maximum: int = MAX_METADATA_BYTES) -> dict: + with path.open("rb") as stream: + data = stream.read(maximum + 1) + if len(data) > maximum: + raise DeliveryError("metadata size limit") + try: + return json.loads(data, object_pairs_hook=unique_object) + except (UnicodeError, json.JSONDecodeError) as error: + raise DeliveryError("invalid JSON metadata") from error + + +def canonical(value: Any) -> bytes: + return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode() + + +def contract(path: Path) -> dict: + data = read_json(path, 16384) + fields( + data, + { + "schema_version", + "format", + "platform", + "toolchains", + "capacity", + "layout", + "services", + "recovery", + }, + "delivery contract", + ) + if type(data["schema_version"]) is not int or data["schema_version"] != 1: + raise DeliveryError("unsupported delivery contract") + if data["format"] != FORMAT or data["platform"] != PLATFORM or data["toolchains"] != TOOLCHAINS: + raise DeliveryError("unsupported delivery target or toolchain") + capacity = fields( + data["capacity"], + {"memory_bytes", "disk_bytes", "execution_slots", "import_slots"}, + "capacity", + ) + for name, value in capacity.items(): + integer(value, 1, 16 * 1024**3, name) + if capacity != { + "memory_bytes": 4 * 1024**3, + "disk_bytes": 12 * 1024**3, + "execution_slots": 1, + "import_slots": 1, + }: + raise DeliveryError("delivery capacity mismatch") + layout = fields( + data["layout"], + { + "releases", + "current", + "configuration", + "data", + "judge_configuration", + "judge_state", + "task_cgroup", + }, + "layout", + ) + if layout != { + "releases": "/opt/kf-onlinejudge/releases", + "current": "/opt/kf-onlinejudge/current", + "configuration": "/etc/kf-onlinejudge", + "data": "/var/lib/kf-onlinejudge", + "judge_configuration": "/etc/kf-judge", + "judge_state": "/var/lib/kf-judge", + "task_cgroup": "/sys/fs/cgroup/kf-judge-tasks", + }: + raise DeliveryError("delivery layout mismatch") + expected_services = { + "api": "platform", + "coordinator": "coordinator", + "outbox": "platform", + "problem-import": "problem_import", + "worker": "none", + "sandbox-factory": "none", + "identity-operator": "identity_operator", + "migration": "migration_owner", + } + if data["services"] != expected_services: + raise DeliveryError("service authority mismatch") + fields( + data["recovery"], {"consistent_cut_rpo_seconds", "prepared_restore_rto_seconds"}, "recovery" + ) + for name, value in data["recovery"].items(): + integer(value, 0, 3600, name) + if data["recovery"] != {"consistent_cut_rpo_seconds": 0, "prepared_restore_rto_seconds": 1800}: + raise DeliveryError("recovery contract mismatch") + return data