feat(delivery): add bounded offline release tooling
This commit is contained in:
@@ -0,0 +1,92 @@
|
||||
"""Pure delivery-contract tests; these do not claim installation or VM safety."""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
policy = importlib.import_module("scripts.release.policy")
|
||||
manifest = importlib.import_module("scripts.release.manifest")
|
||||
architecture = importlib.import_module("scripts.check_architecture")
|
||||
|
||||
|
||||
def test_delivery_authorities_and_layout():
|
||||
contract = policy.contract(ROOT / "deploy/release/contract.v1.json")
|
||||
assert contract["services"]["worker"] == "none"
|
||||
assert contract["services"]["sandbox-factory"] == "none"
|
||||
assert contract["layout"]["task_cgroup"] != contract["layout"]["judge_state"]
|
||||
assert contract["capacity"]["execution_slots"] == 1
|
||||
assert contract["recovery"]["consistent_cut_rpo_seconds"] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [True, 0, 50001, "1"])
|
||||
def test_integer_resource_limits_cannot_be_coerced(value):
|
||||
with pytest.raises(policy.DeliveryError):
|
||||
policy.integer(value, 1, 50000, "entry count")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"docs/a.md",
|
||||
"backend/.env",
|
||||
"backend/../secrets",
|
||||
"/bin/tool",
|
||||
"images/.ssh/id",
|
||||
"site/entry.pyc",
|
||||
],
|
||||
)
|
||||
def test_private_and_noncanonical_paths_are_not_payload(name):
|
||||
with pytest.raises(policy.DeliveryError):
|
||||
policy.relative_name(name)
|
||||
|
||||
|
||||
def test_expected_hash_is_external_and_not_placeholder():
|
||||
for value in ("0" * 64, "a" * 64, "bad"):
|
||||
with pytest.raises(policy.DeliveryError):
|
||||
policy.sha256(value)
|
||||
|
||||
|
||||
def test_no_silent_contract_fields(tmp_path):
|
||||
value = json.loads((ROOT / "deploy/release/contract.v1.json").read_text())
|
||||
value["ignore_isolation"] = True
|
||||
path = tmp_path / "contract.json"
|
||||
path.write_text(json.dumps(value))
|
||||
with pytest.raises(policy.DeliveryError, match="contract fields"):
|
||||
policy.contract(path)
|
||||
|
||||
|
||||
def test_duplicate_metadata_key_is_rejected():
|
||||
with pytest.raises(policy.DeliveryError, match="duplicate"):
|
||||
json.loads(
|
||||
'{"schema_version":1,"schema_version":2}', object_pairs_hook=policy.unique_object
|
||||
)
|
||||
|
||||
|
||||
def test_inventory_of_trusted_read_only_files(tmp_path):
|
||||
directory = tmp_path / "backend"
|
||||
directory.mkdir()
|
||||
file = directory / "version.txt"
|
||||
file.write_bytes(b"accepted-source\n")
|
||||
file.chmod(0o444)
|
||||
directory.chmod(0o555)
|
||||
entries = manifest.inventory(tmp_path)
|
||||
assert entries[0]["path"] == "backend/version.txt"
|
||||
assert entries[0]["bytes"] == 16
|
||||
assert manifest.directories(tmp_path) == ["backend"]
|
||||
|
||||
|
||||
def test_delivery_cannot_import_database_state(tmp_path, monkeypatch):
|
||||
(tmp_path / "bad.py").write_text("from kf_oj.modules.submissions.models import Evaluation\n")
|
||||
monkeypatch.setattr(architecture, "RELEASE", tmp_path)
|
||||
assert len(architecture.release_violations()) == 1
|
||||
|
||||
|
||||
def test_delivery_may_use_public_wire_contracts(tmp_path, monkeypatch):
|
||||
(tmp_path / "public.py").write_text("from kf_oj.contracts.judging_v2 import NodeAdmission\n")
|
||||
monkeypatch.setattr(architecture, "RELEASE", tmp_path)
|
||||
assert architecture.release_violations() == []
|
||||
@@ -0,0 +1,46 @@
|
||||
# Single-node delivery contract
|
||||
|
||||
`contract.v1.json` describes the first pre-release installation interface. It is
|
||||
separate from judging v1/v2 wire formats and never rewrites historical reports.
|
||||
The reference target is Arch/CachyOS Linux x86_64-v3, glibc 2.44, systemd and cgroup
|
||||
v2. Images and service libraries are bound to explicit file/catalog fingerprints;
|
||||
support for other distributions requires a separately verified delivery profile.
|
||||
|
||||
The target has at least 4 GiB RAM and 12 GiB installation/data capacity. Judging
|
||||
and problem-import concurrency remain one each. Configured service and execution
|
||||
budgets, effective kernel limits and free capacity must also pass preflight; total
|
||||
RAM alone does not prove enough capacity or verified execution isolation.
|
||||
|
||||
Release payloads are directory bundles with a bounded manifest: at most 50,000
|
||||
single-link regular files, 256 MiB per file, 4 GiB total and 8 MiB metadata. The
|
||||
manifest SHA-256 is supplied separately by the operator. Links, extra members,
|
||||
private paths and unexpected file changes are refused. Built payloads and real
|
||||
configuration never belong in Git. Validation is read-only and does not claim
|
||||
that any service has been installed, started or verified:
|
||||
|
||||
```sh
|
||||
python3.13 -m scripts.release contract deploy/release/contract.v1.json
|
||||
python3.13 -m scripts.release verify /path/to/bundle --manifest-sha256 SHA256_FROM_RECEIPT
|
||||
```
|
||||
|
||||
Application versions live below `/opt/kf-onlinejudge/releases`; `current` selects
|
||||
an explicitly installed version. Private operator/service configuration lives in
|
||||
`/etc/kf-onlinejudge` and `/etc/kf-judge`, mutable application data under
|
||||
`/var/lib/kf-onlinejudge`, and protected Sandbox state/images under
|
||||
`/var/lib/kf-judge`. The delegated task cgroup is separate from the Root factory's
|
||||
own supervision cgroup. Application processes do not initialize tables on start;
|
||||
Backend Alembic remains the only migration entry point.
|
||||
|
||||
Database login identities are separate for migration, API/outbox, Coordinator,
|
||||
problem import and local identity operations. Rust Worker and Root factory do not
|
||||
hold database credentials. Root factory has no MQ/platform/SSH/GitHub identity;
|
||||
privileged execution remains inside Sandbox. Public and admin browser traffic
|
||||
share the HTTPS site; execution-control interfaces remain internal.
|
||||
|
||||
Upgrade and restore must retain immutable problem versions, submitted source,
|
||||
audit and original judging reports. A consistent backup cut has a zero confirmed
|
||||
data-loss target; restoring from already prepared release/backup materials has a
|
||||
30-minute recovery target, including service health and actual four-language
|
||||
verification. These are acceptance targets, not results inferred from this file.
|
||||
Installation/service/restore evidence is required before this pre-release can be
|
||||
treated as an accepted delivery; the contract itself does not deploy production.
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"format": "kf.single-node.directory.v1",
|
||||
"platform": "linux_x86_64_v3_glibc244_systemd",
|
||||
"toolchains": {"python": "3.13.16", "rust": "1.99.0", "node": "26.10.0", "pnpm": "11.28.4"},
|
||||
"capacity": {"memory_bytes": 4294967296, "disk_bytes": 12884901888, "execution_slots": 1, "import_slots": 1},
|
||||
"layout": {
|
||||
"releases": "/opt/kf-onlinejudge/releases",
|
||||
"current": "/opt/kf-onlinejudge/current",
|
||||
"configuration": "/etc/kf-onlinejudge",
|
||||
"data": "/var/lib/kf-onlinejudge",
|
||||
"judge_configuration": "/etc/kf-judge",
|
||||
"judge_state": "/var/lib/kf-judge",
|
||||
"task_cgroup": "/sys/fs/cgroup/kf-judge-tasks"
|
||||
},
|
||||
"services": {
|
||||
"api": "platform",
|
||||
"coordinator": "coordinator",
|
||||
"outbox": "platform",
|
||||
"problem-import": "problem_import",
|
||||
"worker": "none",
|
||||
"sandbox-factory": "none",
|
||||
"identity-operator": "identity_operator",
|
||||
"migration": "migration_owner"
|
||||
},
|
||||
"recovery": {"consistent_cut_rpo_seconds": 0, "prepared_restore_rto_seconds": 1800}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PACKAGE = ROOT / "backend/src/kf_oj"
|
||||
RELEASE = ROOT / "scripts/release"
|
||||
RUST_ALLOWED = {
|
||||
"kf-contracts": set(),
|
||||
"kf-sandbox": {"kf-contracts"},
|
||||
@@ -97,12 +98,22 @@ def rust_violations() -> list[str]:
|
||||
return violations
|
||||
|
||||
|
||||
def release_violations() -> list[str]:
|
||||
"""Offline delivery may use public contracts, never application state layers."""
|
||||
violations = []
|
||||
for path in sorted(RELEASE.glob("*.py")):
|
||||
for target in imports(ast.parse(path.read_text(encoding="utf-8")), "scripts.release"):
|
||||
if target.startswith("kf_oj.") and not target.startswith("kf_oj.contracts."):
|
||||
violations.append(f"{path.name}: release cannot import application state: {target}")
|
||||
return violations
|
||||
|
||||
|
||||
def main() -> None:
|
||||
violations = python_violations() + rust_violations()
|
||||
violations = python_violations() + rust_violations() + release_violations()
|
||||
if violations:
|
||||
print("\n".join(violations), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
print("Architecture boundaries: OK (Python imports + Rust crate graph)")
|
||||
print("Architecture boundaries: OK (Python imports + Rust crate graph + offline delivery)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
# Offline release tooling
|
||||
|
||||
This package is an operator composition tool. `policy` validates delivery bounds,
|
||||
`manifest` inventories/verifies explicit files, and `build` runs the locked trusted
|
||||
checkout build and copies only tracked application/deployment sources, fresh
|
||||
production dependencies and separately fingerprinted public runtime inputs.
|
||||
It does not import domain ORM/repositories, write application state, implement a
|
||||
second migration path, execute student code or perform Sandbox privilege work.
|
||||
`scripts/check_architecture.py` enforces its application import boundary.
|
||||
|
||||
All dependency resolution for a release uses existing lockfiles and offline
|
||||
caches. Backend source is shipped with locked production dependencies rather
|
||||
than invoking an unpinned isolated wheel build backend. uvx/pip and package
|
||||
startup path hooks are not delivery runtime entry points. Build inputs specify a
|
||||
verified public runtime inventory and the three existing image catalogs; an
|
||||
independent expected SHA-256 binds each input. Real input paths/receipts are local
|
||||
operator data and do not belong in Git.
|
||||
|
||||
```sh
|
||||
python3.13 -m scripts.release build --repo /native/checkout \
|
||||
--inputs /private/build-inputs.json --output /private/new-bundle \
|
||||
--work /private/new-build-work
|
||||
```
|
||||
|
||||
The checkout must be clean and committed, build output/work directories new,
|
||||
and the output filesystem have the required reserve. The resulting receipt
|
||||
contains the source and manifest fingerprints, file count and byte total, with
|
||||
`installed=false` and `execution_verified=false`. Installation and service
|
||||
acceptance remain separate operations. See `deploy/release/README.md` for the
|
||||
versioned installation contract and authority split.
|
||||
@@ -0,0 +1 @@
|
||||
"""Offline delivery tools; no application state or sandbox privilege operations."""
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Operator entry points; validation does not imply installed service health."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from . import build, manifest, policy
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="KF bounded offline delivery")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
check = sub.add_parser("contract")
|
||||
check.add_argument("path", type=Path)
|
||||
check = sub.add_parser("verify")
|
||||
check.add_argument("bundle", type=Path)
|
||||
check.add_argument("--manifest-sha256", required=True)
|
||||
check = sub.add_parser("build")
|
||||
check.add_argument("--repo", type=Path, required=True)
|
||||
check.add_argument("--inputs", type=Path, required=True)
|
||||
check.add_argument("--output", type=Path, required=True)
|
||||
check.add_argument("--work", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.command == "contract":
|
||||
policy.contract(args.path)
|
||||
result = {"contract_valid": True, "installed": False, "execution_verified": False}
|
||||
elif args.command == "verify":
|
||||
data = manifest.verify(args.bundle, args.manifest_sha256)
|
||||
result = {
|
||||
"bundle_valid": True,
|
||||
"source_sha": data["source_sha"],
|
||||
"payload_bytes": data["payload_bytes"],
|
||||
"files": len(data["files"]),
|
||||
"installed": False,
|
||||
"execution_verified": False,
|
||||
}
|
||||
else:
|
||||
result = build.build(args.repo, args.inputs, args.output, args.work)
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
except (policy.DeliveryError, OSError, subprocess.SubprocessError) as error:
|
||||
# OSError filenames can include private operator paths; do not echo them.
|
||||
parser.exit(
|
||||
1,
|
||||
"Delivery refused: "
|
||||
+ (str(error) if isinstance(error, policy.DeliveryError) else type(error).__name__)
|
||||
+ "\n",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,328 @@
|
||||
"""Trusted checkout + hash-bound offline runtime inputs -> directory bundle."""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from . import manifest
|
||||
from .policy import DeliveryError, contract, digest, fields, read_json, relative_name, sha256
|
||||
|
||||
ENTRYPOINTS = {
|
||||
"kf-api": (
|
||||
"uvicorn kf_oj.api.app:create_app --factory --host 127.0.0.1 --port 8000 "
|
||||
"--no-proxy-headers --limit-concurrency 32 --backlog 64"
|
||||
),
|
||||
"kf-outbox-publisher": "kf_oj.workers.outbox",
|
||||
"kf-problem-import-worker": "kf_oj.workers.problem_import",
|
||||
"kf-accounts": "kf_oj.modules.identity.cli",
|
||||
"kf-problems": "kf_oj.modules.problems.cli",
|
||||
"kf-artifacts": "kf_oj.infra.artifacts.cli",
|
||||
"kf-migrate": "alembic -c backend/alembic.ini",
|
||||
}
|
||||
|
||||
|
||||
def command(argv: list[str], cwd: Path, log: Path, env: dict | None = None) -> None:
|
||||
with log.open("ab") as output:
|
||||
output.write(("$ " + " ".join(argv) + "\n").encode())
|
||||
output.flush()
|
||||
subprocess.run(
|
||||
argv,
|
||||
cwd=cwd,
|
||||
env=env,
|
||||
stdout=output,
|
||||
stderr=subprocess.STDOUT,
|
||||
check=True,
|
||||
timeout=1200,
|
||||
)
|
||||
|
||||
|
||||
def copy(source: Path, target: Path) -> None:
|
||||
info = source.stat()
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_size > manifest.MAX_FILE_BYTES:
|
||||
raise DeliveryError("unsupported source payload file")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with source.open("rb") as incoming, target.open("xb") as outgoing:
|
||||
shutil.copyfileobj(incoming, outgoing)
|
||||
target.chmod(0o555 if info.st_mode & 0o111 else 0o444)
|
||||
|
||||
|
||||
def tree(source: Path, target: Path) -> None:
|
||||
if source.is_symlink() or not source.is_dir():
|
||||
raise DeliveryError("ordinary source directory required")
|
||||
for directory, directories, names in os.walk(source, followlinks=False):
|
||||
directories[:] = sorted(name for name in directories if name not in {"__pycache__", ".git"})
|
||||
for name in directories:
|
||||
if (Path(directory) / name).is_symlink():
|
||||
raise DeliveryError("source directory link refused")
|
||||
for name in sorted(names):
|
||||
path = Path(directory) / name
|
||||
if path.is_symlink():
|
||||
raise DeliveryError("source file link refused")
|
||||
if name.endswith((".pyc", ".pyo")) or name in {"_virtualenv.py", "_virtualenv.pth"}:
|
||||
continue
|
||||
if name.endswith(".pth"):
|
||||
raise DeliveryError("dependency startup path hook refused")
|
||||
relative = path.relative_to(source)
|
||||
relative_name(
|
||||
(target.name + "/" + relative.as_posix())
|
||||
if target.name in {"site", "frontend"}
|
||||
else "backend/" + relative.as_posix()
|
||||
)
|
||||
copy(path, target / relative)
|
||||
|
||||
|
||||
def tracked(repo: Path, source_prefix: str, payload: Path) -> None:
|
||||
"""Only tracked checkout members; never scan operator environment files."""
|
||||
names = subprocess.check_output(["git", "ls-files", "-z", source_prefix], cwd=repo)
|
||||
for raw in names.split(b"\0"):
|
||||
if not raw:
|
||||
continue
|
||||
name = raw.decode()
|
||||
relative_name(name)
|
||||
source = repo / name
|
||||
if source.is_symlink():
|
||||
raise DeliveryError("tracked payload link refused")
|
||||
copy(source, payload / name)
|
||||
|
||||
|
||||
def curated(root: Path, records: dict, name: str, target: Path) -> None:
|
||||
"""Resolve only links explicitly bound by the separate trusted inventory."""
|
||||
current = root / name
|
||||
for _ in range(9):
|
||||
relative = current.relative_to(root).as_posix()
|
||||
entry = records.get(relative)
|
||||
if entry is None:
|
||||
raise DeliveryError("runtime input absent from trusted inventory")
|
||||
if "link" not in entry:
|
||||
if (
|
||||
current.is_symlink()
|
||||
or current.stat().st_size != entry["bytes"]
|
||||
or digest(current) != entry["sha256"]
|
||||
):
|
||||
raise DeliveryError("runtime input fingerprint differs")
|
||||
copy(current, target)
|
||||
return
|
||||
if not current.is_symlink() or os.readlink(current) != entry["link"]:
|
||||
raise DeliveryError("runtime input link differs")
|
||||
if entry["link"].startswith("/"):
|
||||
raise DeliveryError("absolute runtime input link refused")
|
||||
current = Path(os.path.normpath(current.parent / entry["link"]))
|
||||
if not current.is_relative_to(root):
|
||||
raise DeliveryError("runtime input escapes trusted root")
|
||||
raise DeliveryError("runtime input link depth limit")
|
||||
|
||||
|
||||
def image(source: Path, expected: str, target: Path) -> dict:
|
||||
sha256(expected)
|
||||
if source.is_symlink() or digest(source / "manifest.json") != expected:
|
||||
raise DeliveryError("runtime catalog fingerprint differs")
|
||||
catalog = read_json(source / "manifest.json")
|
||||
identity = {
|
||||
name: catalog[name]
|
||||
for name in ("toolchain_id", "image_kind", "image_revision", "runtime_version")
|
||||
}
|
||||
identity["image_digest"] = expected
|
||||
if catalog.get("format_version") != 2 or catalog.get("architecture") != "x86_64_v3":
|
||||
raise DeliveryError("unsupported image catalog")
|
||||
records = catalog.get("files")
|
||||
directories = catalog.get("directories")
|
||||
if (
|
||||
not isinstance(records, list)
|
||||
or not 1 <= len(records) <= 10000
|
||||
or not isinstance(directories, list)
|
||||
or len(directories) > 10000
|
||||
):
|
||||
raise DeliveryError("image catalog bounds")
|
||||
target.mkdir(parents=True)
|
||||
for name in directories:
|
||||
relative_name("images/catalog/" + name + "/.__directory__")
|
||||
original = source / name
|
||||
if original.is_symlink() or not original.is_dir():
|
||||
raise DeliveryError("runtime directory differs from catalog")
|
||||
(target / name).mkdir(parents=True, exist_ok=True)
|
||||
for item in records:
|
||||
name = item["path"]
|
||||
relative_name("images/catalog/" + name)
|
||||
original = source / name
|
||||
if (
|
||||
original.is_symlink()
|
||||
or original.stat().st_size != item["bytes"]
|
||||
or digest(original) != item["sha256"]
|
||||
):
|
||||
raise DeliveryError("runtime file differs from catalog")
|
||||
copy(original, target / name)
|
||||
copy(source / "manifest.json", target / "manifest.json")
|
||||
return identity
|
||||
|
||||
|
||||
def build(repo: Path, inputs: Path, output: Path, work: Path) -> dict:
|
||||
repo = repo.resolve(strict=True)
|
||||
if subprocess.check_output(["git", "status", "--porcelain"], cwd=repo):
|
||||
raise DeliveryError("build requires a clean committed checkout")
|
||||
source_sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip()
|
||||
contract(repo / "deploy/release/contract.v1.json")
|
||||
fields_spec = {
|
||||
"schema_version",
|
||||
"runtime_root",
|
||||
"runtime_manifest",
|
||||
"runtime_manifest_sha256",
|
||||
"images",
|
||||
}
|
||||
spec = fields(read_json(inputs, 16384), fields_spec, "build inputs")
|
||||
if type(spec["schema_version"]) is not int or spec["schema_version"] != 1:
|
||||
raise DeliveryError("unsupported build inputs")
|
||||
root = Path(spec["runtime_root"]).resolve(strict=True)
|
||||
trusted_manifest = Path(spec["runtime_manifest"])
|
||||
if digest(trusted_manifest) != sha256(spec["runtime_manifest_sha256"]):
|
||||
raise DeliveryError("runtime inventory fingerprint differs")
|
||||
runtime_records = read_json(trusted_manifest).get("files")
|
||||
if not isinstance(runtime_records, list) or len(runtime_records) > 50000:
|
||||
raise DeliveryError("runtime inventory bounds")
|
||||
records = {entry["path"]: entry for entry in runtime_records}
|
||||
if len(records) != len(runtime_records):
|
||||
raise DeliveryError("duplicate runtime inventory member")
|
||||
fields(spec["images"], {"gnu", "python313", "java21"}, "build images")
|
||||
if output.exists() or output.is_symlink() or work.exists() or work.is_symlink():
|
||||
raise DeliveryError("build output/work directories must be new")
|
||||
if shutil.disk_usage(output.parent).free < 8 * 1024**3:
|
||||
raise DeliveryError("build disk reserve below 8 GiB")
|
||||
output.mkdir(mode=0o700)
|
||||
work.mkdir(mode=0o700)
|
||||
payload = output / "payload"
|
||||
payload.mkdir()
|
||||
locks = {
|
||||
name: digest(repo / name)
|
||||
for name in (
|
||||
"backend/uv.lock",
|
||||
"judge/Cargo.lock",
|
||||
"frontend/pnpm-lock.yaml",
|
||||
"deploy/release/contract.v1.json",
|
||||
)
|
||||
}
|
||||
log = work / "build.log"
|
||||
env = os.environ.copy()
|
||||
env["UV_PROJECT_ENVIRONMENT"] = str(work / "venv")
|
||||
env["CARGO_BUILD_JOBS"] = "2"
|
||||
for argv, cwd in [
|
||||
(
|
||||
[
|
||||
"uv",
|
||||
"sync",
|
||||
"--locked",
|
||||
"--no-dev",
|
||||
"--no-install-project",
|
||||
"--no-build",
|
||||
"--offline",
|
||||
"--no-python-downloads",
|
||||
"--python",
|
||||
"3.13.16",
|
||||
],
|
||||
repo / "backend",
|
||||
),
|
||||
(
|
||||
["pnpm", "install", "--frozen-lockfile", "--offline", "--ignore-scripts"],
|
||||
repo / "frontend",
|
||||
),
|
||||
(["pnpm", "build"], repo / "frontend"),
|
||||
(
|
||||
[
|
||||
str(Path.home() / ".cargo/bin/cargo"),
|
||||
"build",
|
||||
"--release",
|
||||
"--workspace",
|
||||
"--locked",
|
||||
"--offline",
|
||||
],
|
||||
repo / "judge",
|
||||
),
|
||||
]:
|
||||
command(argv, cwd, log, env)
|
||||
command([str(work / "venv/bin/python"), "--version"], repo, log)
|
||||
command(["node", "--version"], repo / "frontend", log)
|
||||
command(["pnpm", "--version"], repo / "frontend", log)
|
||||
command([str(Path.home() / ".cargo/bin/rustc"), "--version"], repo / "judge", log)
|
||||
actual = {
|
||||
"python": subprocess.check_output([str(work / "venv/bin/python"), "--version"], text=True)
|
||||
.strip()
|
||||
.removeprefix("Python "),
|
||||
"node": subprocess.check_output(["node", "--version"], cwd=repo / "frontend", text=True)
|
||||
.strip()
|
||||
.removeprefix("v"),
|
||||
"pnpm": subprocess.check_output(
|
||||
["pnpm", "--version"], cwd=repo / "frontend", text=True
|
||||
).strip(),
|
||||
"rust": subprocess.check_output(
|
||||
[str(Path.home() / ".cargo/bin/rustc"), "--version"], cwd=repo / "judge", text=True
|
||||
).split()[1],
|
||||
}
|
||||
if actual != manifest.TOOLCHAINS:
|
||||
raise DeliveryError("actual build toolchain differs from contract")
|
||||
tracked(repo, "backend/src", payload)
|
||||
tracked(repo, "backend/migrations", payload)
|
||||
copy(repo / "backend/alembic.ini", payload / "backend/alembic.ini")
|
||||
tree(work / "venv/lib/python3.13/site-packages", payload / "site")
|
||||
tree(repo / "frontend/dist", payload / "frontend")
|
||||
tracked(repo, "deploy", payload)
|
||||
for name in ("kf-sandbox", "kf-worker", "kf-coordinator"):
|
||||
copy(repo / "judge/target/release" / name, payload / "bin" / name)
|
||||
for name in sorted(records):
|
||||
if name.startswith("python/lib/") or name == "python/bin/python3.13":
|
||||
if name.endswith((".pyc", ".pyo", ".a")) or any(
|
||||
p
|
||||
in {
|
||||
"site-packages",
|
||||
"__pycache__",
|
||||
"test",
|
||||
"tests",
|
||||
"ensurepip",
|
||||
"tkinter",
|
||||
"idlelib",
|
||||
}
|
||||
for p in Path(name).parts
|
||||
):
|
||||
continue
|
||||
relative_name(name)
|
||||
curated(root, records, name, payload / name)
|
||||
elif name.startswith("services/usr/lib/"):
|
||||
relative = name.removeprefix("services/usr/lib/")
|
||||
if "/" not in relative and (".so" in relative or relative == "ld-linux-x86-64.so.2"):
|
||||
relative_name("lib/" + relative)
|
||||
curated(root, records, name, payload / "lib" / relative)
|
||||
images = {}
|
||||
for name, entry in spec["images"].items():
|
||||
fields(entry, {"root", "catalog_sha256"}, "image build input")
|
||||
images[name] = image(
|
||||
Path(entry["root"]), entry["catalog_sha256"], payload / "images" / name
|
||||
)
|
||||
for name, module in ENTRYPOINTS.items():
|
||||
wrapper = (
|
||||
"#!/bin/sh\nset -eu\n"
|
||||
'APP_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd -P)\n'
|
||||
'export PYTHONHOME="$APP_ROOT/python"\n'
|
||||
'export PYTHONPATH="$APP_ROOT/backend/src:$APP_ROOT/site"\n'
|
||||
"export PYTHONDONTWRITEBYTECODE=1\n"
|
||||
'export LD_LIBRARY_PATH="$APP_ROOT/lib:$APP_ROOT/python/lib"\n'
|
||||
'cd "$APP_ROOT"\n'
|
||||
'exec "$APP_ROOT/python/bin/python3.13" -s -P -m ' + module + ' "$@"\n'
|
||||
)
|
||||
target = payload / "bin" / name
|
||||
target.write_text(wrapper)
|
||||
target.chmod(0o555)
|
||||
for directory, directories, _ in os.walk(payload, topdown=False):
|
||||
for name in directories:
|
||||
(Path(directory) / name).chmod(0o555)
|
||||
payload.chmod(0o555)
|
||||
if subprocess.check_output(["git", "status", "--porcelain"], cwd=repo):
|
||||
raise DeliveryError("source changed during build")
|
||||
data = manifest.write(output, source_sha, locks, images)
|
||||
manifest.verify(output, digest(output / "manifest.json"))
|
||||
return {
|
||||
"source_sha": source_sha,
|
||||
"manifest_sha256": digest(output / "manifest.json"),
|
||||
"payload_bytes": data["payload_bytes"],
|
||||
"files": len(data["files"]),
|
||||
"installed": False,
|
||||
"execution_verified": False,
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
"""Explicit directory bundle inventories, verified before installation."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
from pathlib import Path
|
||||
|
||||
from .policy import (
|
||||
FORMAT,
|
||||
MAX_BUNDLE_BYTES,
|
||||
MAX_FILE_BYTES,
|
||||
MAX_FILES,
|
||||
MAX_METADATA_BYTES,
|
||||
PLATFORM,
|
||||
TOOLCHAINS,
|
||||
DeliveryError,
|
||||
canonical,
|
||||
digest,
|
||||
fields,
|
||||
integer,
|
||||
read_json,
|
||||
relative_name,
|
||||
sha256,
|
||||
)
|
||||
|
||||
|
||||
def inventory(root: Path) -> list[dict]:
|
||||
"""No links/devices or inferred inclusion; bounds apply before hashing."""
|
||||
if not root.is_dir() or root.is_symlink():
|
||||
raise DeliveryError("payload root must be an ordinary directory")
|
||||
entries = []
|
||||
total = 0
|
||||
directory_count = 0
|
||||
for directory, directories, files in os.walk(root, followlinks=False):
|
||||
directory_count += len(directories)
|
||||
if directory_count + len(entries) + len(files) > MAX_FILES:
|
||||
raise DeliveryError("payload entry count limit")
|
||||
for name in directories:
|
||||
path = Path(directory) / name
|
||||
if path.is_symlink():
|
||||
raise DeliveryError("payload directory link refused")
|
||||
for name in sorted(files):
|
||||
path = Path(directory) / name
|
||||
info = path.lstat()
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
|
||||
raise DeliveryError("payload must contain single-link regular files")
|
||||
if stat.S_IMODE(info.st_mode) not in (0o444, 0o555):
|
||||
raise DeliveryError("payload file must be read-only")
|
||||
size = integer(info.st_size, 0, MAX_FILE_BYTES, "payload file size")
|
||||
total += size
|
||||
if total > MAX_BUNDLE_BYTES or len(entries) >= MAX_FILES:
|
||||
raise DeliveryError("payload aggregate limit")
|
||||
relative = relative_name(path.relative_to(root).as_posix())
|
||||
entries.append(
|
||||
{
|
||||
"path": relative,
|
||||
"bytes": size,
|
||||
"sha256": digest(path),
|
||||
"mode": "0555" if info.st_mode & 0o111 else "0444",
|
||||
}
|
||||
)
|
||||
return sorted(entries, key=lambda x: x["path"])
|
||||
|
||||
|
||||
def directories(root: Path) -> list[str]:
|
||||
result = []
|
||||
for directory, names, _ in os.walk(root, followlinks=False):
|
||||
for name in names:
|
||||
path = Path(directory) / name
|
||||
value = path.relative_to(root).as_posix()
|
||||
relative_name(value + "/.__directory__")
|
||||
if path.is_symlink() or stat.S_IMODE(path.stat().st_mode) != 0o555:
|
||||
raise DeliveryError("payload directories must be read-only ordinary directories")
|
||||
result.append(value)
|
||||
if len(result) > MAX_FILES:
|
||||
raise DeliveryError("payload directory count limit")
|
||||
return sorted(result)
|
||||
|
||||
|
||||
def validate(data: dict) -> dict:
|
||||
fields(
|
||||
data,
|
||||
{
|
||||
"schema_version",
|
||||
"format",
|
||||
"release_id",
|
||||
"source_sha",
|
||||
"platform",
|
||||
"toolchains",
|
||||
"locks",
|
||||
"images",
|
||||
"files",
|
||||
"directories",
|
||||
"payload_bytes",
|
||||
},
|
||||
"bundle",
|
||||
)
|
||||
if (
|
||||
type(data["schema_version"]) is not int
|
||||
or data["schema_version"] != 1
|
||||
or data["format"] != FORMAT
|
||||
):
|
||||
raise DeliveryError("unsupported bundle format")
|
||||
if not isinstance(data["source_sha"], str) or not re.fullmatch(
|
||||
r"[0-9a-f]{40}", data["source_sha"]
|
||||
):
|
||||
raise DeliveryError("invalid source SHA")
|
||||
if (
|
||||
not isinstance(data["release_id"], str)
|
||||
or data["release_id"] != "sd-" + data["source_sha"][:12]
|
||||
):
|
||||
raise DeliveryError("release identity mismatch")
|
||||
if data["platform"] != PLATFORM or data["toolchains"] != TOOLCHAINS:
|
||||
raise DeliveryError("bundle platform/toolchain mismatch")
|
||||
fields(
|
||||
data["locks"],
|
||||
{
|
||||
"backend/uv.lock",
|
||||
"judge/Cargo.lock",
|
||||
"frontend/pnpm-lock.yaml",
|
||||
"deploy/release/contract.v1.json",
|
||||
},
|
||||
"lock fingerprints",
|
||||
)
|
||||
for value in data["locks"].values():
|
||||
sha256(value)
|
||||
fields(data["images"], {"gnu", "python313", "java21"}, "runtime images")
|
||||
for image in data["images"].values():
|
||||
fields(
|
||||
image,
|
||||
{"toolchain_id", "image_kind", "image_revision", "runtime_version", "image_digest"},
|
||||
"runtime identity",
|
||||
)
|
||||
sha256(image["image_digest"])
|
||||
integer(image["image_revision"], 1, 2**32 - 1, "image revision")
|
||||
for name in ("toolchain_id", "image_kind", "runtime_version"):
|
||||
if not isinstance(image[name], str) or not re.fullmatch(
|
||||
r"[A-Za-z0-9_.+-]{1,128}", image[name]
|
||||
):
|
||||
raise DeliveryError("invalid runtime identity")
|
||||
runtimes = {
|
||||
"gnu": ("gnu-cxx-v1", "linux_gnu_cxx_x86_64_v1", None),
|
||||
"python313": ("cpython313-v1", "linux_cpython313_x86_64_v1", "3.13.16"),
|
||||
"java21": ("openjdk21-v1", "linux_openjdk21_x86_64_v1", "21.0.12.1+1-LTS"),
|
||||
}
|
||||
for name, (toolchain, kind, version) in runtimes.items():
|
||||
image = data["images"][name]
|
||||
if (
|
||||
image["toolchain_id"] != toolchain
|
||||
or image["image_kind"] != kind
|
||||
or (version is not None and image["runtime_version"] != version)
|
||||
):
|
||||
raise DeliveryError("unsupported runtime image")
|
||||
files = data["files"]
|
||||
if not isinstance(files, list) or not 1 <= len(files) <= MAX_FILES:
|
||||
raise DeliveryError("bundle file count")
|
||||
seen = set()
|
||||
total = 0
|
||||
for entry in files:
|
||||
fields(entry, {"path", "bytes", "sha256", "mode"}, "bundle file")
|
||||
path = relative_name(entry["path"])
|
||||
if path in seen or entry["mode"] not in {"0444", "0555"}:
|
||||
raise DeliveryError("duplicate file or unsafe mode")
|
||||
seen.add(path)
|
||||
total += integer(entry["bytes"], 0, MAX_FILE_BYTES, "bundle file size")
|
||||
sha256(entry["sha256"])
|
||||
if (
|
||||
total > MAX_BUNDLE_BYTES
|
||||
or type(data["payload_bytes"]) is not int
|
||||
or data["payload_bytes"] != total
|
||||
):
|
||||
raise DeliveryError("bundle byte total")
|
||||
required = {
|
||||
"bin/kf-sandbox",
|
||||
"bin/kf-worker",
|
||||
"bin/kf-coordinator",
|
||||
"python/bin/python3.13",
|
||||
"backend/alembic.ini",
|
||||
"frontend/index.html",
|
||||
"frontend/admin/index.html",
|
||||
"deploy/release/contract.v1.json",
|
||||
}
|
||||
required.update(f"images/{name}/manifest.json" for name in data["images"])
|
||||
if not required <= seen:
|
||||
raise DeliveryError("incomplete application bundle")
|
||||
directory_names = data["directories"]
|
||||
if not isinstance(directory_names, list) or len(directory_names) + len(files) > MAX_FILES:
|
||||
raise DeliveryError("bundle directory count")
|
||||
directory_set = set()
|
||||
for name in directory_names:
|
||||
if not isinstance(name, str):
|
||||
raise DeliveryError("invalid bundle directory")
|
||||
relative_name(name + "/.__directory__")
|
||||
if name in directory_set or name in seen:
|
||||
raise DeliveryError("duplicate bundle directory")
|
||||
directory_set.add(name)
|
||||
for name in seen | directory_set:
|
||||
for parent in Path(name).parents:
|
||||
if parent.as_posix() != "." and parent.as_posix() not in directory_set:
|
||||
raise DeliveryError("missing payload parent directory")
|
||||
return data
|
||||
|
||||
|
||||
def verify(bundle: Path, expected_sha256: str) -> dict:
|
||||
"""The manifest digest comes from a separate trusted operator receipt."""
|
||||
expected_sha256 = sha256(expected_sha256)
|
||||
if not bundle.is_dir() or bundle.is_symlink():
|
||||
raise DeliveryError("bundle directory required")
|
||||
metadata = bundle / "manifest.json"
|
||||
info = metadata.lstat()
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_size > MAX_METADATA_BYTES:
|
||||
raise DeliveryError("unsafe manifest file")
|
||||
if digest(metadata) != expected_sha256:
|
||||
raise DeliveryError("bundle manifest fingerprint mismatch")
|
||||
if set(p.name for p in bundle.iterdir()) != {"manifest.json", "payload"}:
|
||||
raise DeliveryError("unexpected bundle member")
|
||||
data = validate(read_json(metadata))
|
||||
if inventory(bundle / "payload") != sorted(data["files"], key=lambda x: x["path"]):
|
||||
raise DeliveryError("bundle contents differ from manifest")
|
||||
if directories(bundle / "payload") != sorted(data["directories"]):
|
||||
raise DeliveryError("bundle directories differ from manifest")
|
||||
for name, identity in data["images"].items():
|
||||
if digest(bundle / "payload/images" / name / "manifest.json") != identity["image_digest"]:
|
||||
raise DeliveryError("image catalog differs from runtime identity")
|
||||
if (
|
||||
digest(bundle / "payload/deploy/release/contract.v1.json")
|
||||
!= data["locks"]["deploy/release/contract.v1.json"]
|
||||
):
|
||||
raise DeliveryError("delivery contract fingerprint mismatch")
|
||||
return data
|
||||
|
||||
|
||||
def write(bundle: Path, source_sha: str, locks: dict, images: dict) -> dict:
|
||||
files = inventory(bundle / "payload")
|
||||
data = validate(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"format": FORMAT,
|
||||
"release_id": "sd-" + source_sha[:12],
|
||||
"source_sha": source_sha,
|
||||
"platform": PLATFORM,
|
||||
"toolchains": TOOLCHAINS,
|
||||
"locks": locks,
|
||||
"images": images,
|
||||
"files": files,
|
||||
"directories": directories(bundle / "payload"),
|
||||
"payload_bytes": sum(x["bytes"] for x in files),
|
||||
}
|
||||
)
|
||||
raw = canonical(data)
|
||||
if len(raw) > MAX_METADATA_BYTES:
|
||||
raise DeliveryError("bundle manifest exceeds metadata limit")
|
||||
with (bundle / "manifest.json").open("xb") as stream:
|
||||
stream.write(raw)
|
||||
(bundle / "manifest.json").chmod(0o444)
|
||||
return data
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Versioned delivery bounds, separate from the judging wire protocols."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
MAX_METADATA_BYTES = 8 * 1024 * 1024
|
||||
MAX_FILES = 50000
|
||||
MAX_FILE_BYTES = 256 * 1024 * 1024
|
||||
MAX_BUNDLE_BYTES = 4 * 1024**3
|
||||
FORMAT = "kf.single-node.directory.v1"
|
||||
PLATFORM = "linux_x86_64_v3_glibc244_systemd"
|
||||
TOOLCHAINS = {"python": "3.13.16", "rust": "1.99.0", "node": "26.10.0", "pnpm": "11.28.4"}
|
||||
ROOTS = ("backend/", "python/", "site/", "lib/", "bin/", "frontend/", "images/", "deploy/")
|
||||
FORBIDDEN = {".git", ".localplan", "docs", "__pycache__", ".env", ".ssh"}
|
||||
|
||||
|
||||
class DeliveryError(ValueError):
|
||||
"""An intentionally bounded, non-secret error for the operator."""
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
with path.open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def sha256(value: Any) -> str:
|
||||
if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value):
|
||||
raise DeliveryError("invalid SHA-256")
|
||||
if value == "a" * 64 or value == "0" * 64:
|
||||
raise DeliveryError("placeholder SHA-256 refused")
|
||||
return value
|
||||
|
||||
|
||||
def fields(value: Any, expected: set[str], label: str) -> dict:
|
||||
if not isinstance(value, dict) or set(value) != expected:
|
||||
raise DeliveryError(f"invalid {label} fields")
|
||||
return value
|
||||
|
||||
|
||||
def integer(value: Any, lower: int, upper: int, label: str) -> int:
|
||||
if type(value) is not int or not lower <= value <= upper:
|
||||
raise DeliveryError(f"invalid {label}")
|
||||
return value
|
||||
|
||||
|
||||
def relative_name(value: Any) -> str:
|
||||
if not isinstance(value, str) or len(value) > 512:
|
||||
raise DeliveryError("invalid payload path")
|
||||
if not all(re.fullmatch(r"[A-Za-z0-9_.+@-]{1,255}", p) for p in value.split("/")):
|
||||
raise DeliveryError("noncanonical payload path")
|
||||
if any(p in {".", "..", *FORBIDDEN} for p in value.split("/")):
|
||||
raise DeliveryError("private or unsafe payload path")
|
||||
if not value.startswith(ROOTS) or value.endswith((".pyc", ".pyo")):
|
||||
raise DeliveryError("payload path outside delivery allowlist")
|
||||
return value
|
||||
|
||||
|
||||
def unique_object(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise DeliveryError("duplicate JSON key")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def read_json(path: Path, maximum: int = MAX_METADATA_BYTES) -> dict:
|
||||
with path.open("rb") as stream:
|
||||
data = stream.read(maximum + 1)
|
||||
if len(data) > maximum:
|
||||
raise DeliveryError("metadata size limit")
|
||||
try:
|
||||
return json.loads(data, object_pairs_hook=unique_object)
|
||||
except (UnicodeError, json.JSONDecodeError) as error:
|
||||
raise DeliveryError("invalid JSON metadata") from error
|
||||
|
||||
|
||||
def canonical(value: Any) -> bytes:
|
||||
return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode()
|
||||
|
||||
|
||||
def contract(path: Path) -> dict:
|
||||
data = read_json(path, 16384)
|
||||
fields(
|
||||
data,
|
||||
{
|
||||
"schema_version",
|
||||
"format",
|
||||
"platform",
|
||||
"toolchains",
|
||||
"capacity",
|
||||
"layout",
|
||||
"services",
|
||||
"recovery",
|
||||
},
|
||||
"delivery contract",
|
||||
)
|
||||
if type(data["schema_version"]) is not int or data["schema_version"] != 1:
|
||||
raise DeliveryError("unsupported delivery contract")
|
||||
if data["format"] != FORMAT or data["platform"] != PLATFORM or data["toolchains"] != TOOLCHAINS:
|
||||
raise DeliveryError("unsupported delivery target or toolchain")
|
||||
capacity = fields(
|
||||
data["capacity"],
|
||||
{"memory_bytes", "disk_bytes", "execution_slots", "import_slots"},
|
||||
"capacity",
|
||||
)
|
||||
for name, value in capacity.items():
|
||||
integer(value, 1, 16 * 1024**3, name)
|
||||
if capacity != {
|
||||
"memory_bytes": 4 * 1024**3,
|
||||
"disk_bytes": 12 * 1024**3,
|
||||
"execution_slots": 1,
|
||||
"import_slots": 1,
|
||||
}:
|
||||
raise DeliveryError("delivery capacity mismatch")
|
||||
layout = fields(
|
||||
data["layout"],
|
||||
{
|
||||
"releases",
|
||||
"current",
|
||||
"configuration",
|
||||
"data",
|
||||
"judge_configuration",
|
||||
"judge_state",
|
||||
"task_cgroup",
|
||||
},
|
||||
"layout",
|
||||
)
|
||||
if layout != {
|
||||
"releases": "/opt/kf-onlinejudge/releases",
|
||||
"current": "/opt/kf-onlinejudge/current",
|
||||
"configuration": "/etc/kf-onlinejudge",
|
||||
"data": "/var/lib/kf-onlinejudge",
|
||||
"judge_configuration": "/etc/kf-judge",
|
||||
"judge_state": "/var/lib/kf-judge",
|
||||
"task_cgroup": "/sys/fs/cgroup/kf-judge-tasks",
|
||||
}:
|
||||
raise DeliveryError("delivery layout mismatch")
|
||||
expected_services = {
|
||||
"api": "platform",
|
||||
"coordinator": "coordinator",
|
||||
"outbox": "platform",
|
||||
"problem-import": "problem_import",
|
||||
"worker": "none",
|
||||
"sandbox-factory": "none",
|
||||
"identity-operator": "identity_operator",
|
||||
"migration": "migration_owner",
|
||||
}
|
||||
if data["services"] != expected_services:
|
||||
raise DeliveryError("service authority mismatch")
|
||||
fields(
|
||||
data["recovery"], {"consistent_cut_rpo_seconds", "prepared_restore_rto_seconds"}, "recovery"
|
||||
)
|
||||
for name, value in data["recovery"].items():
|
||||
integer(value, 0, 3600, name)
|
||||
if data["recovery"] != {"consistent_cut_rpo_seconds": 0, "prepared_restore_rto_seconds": 1800}:
|
||||
raise DeliveryError("recovery contract mismatch")
|
||||
return data
|
||||
Reference in New Issue
Block a user