diff --git a/backend/tests/test_delivery.py b/backend/tests/test_delivery.py index 1ae7f60..6ab7690 100644 --- a/backend/tests/test_delivery.py +++ b/backend/tests/test_delivery.py @@ -365,7 +365,8 @@ def test_ordinary_units_bind_exact_batch_without_inline_secrets(): assert "PrivateNetwork=yes" in units["kf-problem-import.service"].decode() assert "PrivatePIDs=yes" in units["kf-problem-import.service"].decode() assert "RestrictAddressFamilies=AF_UNIX\n" in units["kf-problem-import.service"].decode() - assert "-m kf_oj.infra.messaging.publisher" in units["kf-outbox.service"].decode() + assert "-m scripts.release.service_bootstrap outbox" in units["kf-outbox.service"].decode() + assert service_bootstrap.PYTHON_MODULES["outbox"] == ["kf_oj.infra.messaging.publisher"] assert "LoadCredential=worker.json:" in units["kf-worker.service"].decode() @@ -385,3 +386,71 @@ def test_systemd_credential_bytes_are_checked_before_private_copy(tmp_path): path.write_bytes(b"x" * 32769) with pytest.raises(policy.DeliveryError, match="bounded systemd credential"): service_bootstrap.read_credential(path, expected, os.getuid()) + + +def test_import_configuration_has_the_same_required_admission(delivery_configuration): + _, _, data, node, endpoints, _ = delivery_configuration + raw = configuration.compose(ROOT, data, node, endpoints, attest=False) + api = configuration.environment_read(raw["api.env"]) + importer = configuration.environment_read(raw["import.env"]) + assert importer["KF_OJ_JUDGE_ADMISSION"] == api["KF_OJ_JUDGE_ADMISSION"] + assert set(importer) == service_bootstrap.ROLE_ENVIRONMENT["problem-import"] + + +def ordinary_observation(role): + user, memory, pids, _, _, _ = services.PROFILES[role] + uid = str(service_bootstrap.preflight.ACCOUNTS[user]) + return { + "cgroup": "/sys/fs/cgroup/kfonlinejudge.slice/kf-" + role + ".service", + "uids": [uid] * 4, + "gids": [uid] * 4, + "groups": [uid], + "no_new_privileges": "1", + "capabilities": ["0000000000000000"] * 5, + "pid": 1, + "service_limits": { + "memory.max": str(memory * 1024**2), + "memory.swap.max": "0", + "pids.max": str(pids), + "cpu.max": "100000 100000", + }, + "node_limits": { + "memory.max": str(3840 * 1024**2), + "memory.swap.max": "0", + "pids.max": "384", + "cpu.max": "200000 100000", + }, + "process_limits": { + "core": [0, 0], + "file": [64 * 1024**2] * 2, + "files": [64 if role == "problem-import" else 256] * 2, + "stack": [16 * 1024**2] * 2, + }, + "interfaces": ["lo"] if role == "problem-import" else None, + } + + +@pytest.mark.parametrize("role", services.PROFILES) +def test_ordinary_guard_parser_requires_effective_bounds(role): + observed = ordinary_observation(role) + service_bootstrap.validate_runtime(role, observed) + observed["node_limits"]["memory.max"] = "max" + with pytest.raises(policy.DeliveryError, match="effective ordinary/node"): + service_bootstrap.validate_runtime(role, observed) + + +@pytest.mark.parametrize( + "field,value", + [ + ("pid", 43), + ("no_new_privileges", "0"), + ("groups", ["1200", "0"]), + ("capabilities", ["0000000000000001"] * 5), + ("interfaces", ["lo", "eth0"]), + ], +) +def test_import_guard_refuses_missing_pid_privilege_or_network_boundary(field, value): + observed = ordinary_observation("problem-import") + observed[field] = value + with pytest.raises(policy.DeliveryError): + service_bootstrap.validate_runtime("problem-import", observed) diff --git a/scripts/release/README.md b/scripts/release/README.md index d3065f1..82e7659 100644 --- a/scripts/release/README.md +++ b/scripts/release/README.md @@ -121,9 +121,19 @@ API/outbox/import share the established content-storage identity; individual PID namespaces prevent same-UID inspection of another service's environment. This requires the fixed systemd profile and verified kernel namespace support. Rust Coordinator/Worker use distinct ordinary identities and systemd credentials. -`service_bootstrap` verifies those bounded credential bytes, copies them to a +Every ordinary launch checks effective service/node cgroup memory, swap, CPU and +PID limits, file/descriptor/stack limits, UID/GID and absence of capabilities +before exec. The three Python services also require PID 1 in their private PID +namespace; import requires a loopback-only network view. Ignored namespace +directives therefore refuse startup. These checks still require actual VM +verification; they do not attest Sandbox isolation or service readiness. +The import environment includes the same validated admission policy as API. + +`service_bootstrap` verifies bounded Rust credential bytes, copies them to a private service runtime directory with the existing required 0600 mode and then launches only the fixed ordinary binary. It accepts no student executable, command override or privileged Sandbox operation. Tokens/URLs are not command arguments or unit text. The outbox unit uses the existing actual confirmed publisher module; the old scaffold check entry is not treated as a delivery loop. +Python launches use a fixed module/argument list and only the generated service +environment fields; inherited debugging/proxy/provider credentials are omitted. diff --git a/scripts/release/configuration.py b/scripts/release/configuration.py index 42dab84..076f7ba 100644 --- a/scripts/release/configuration.py +++ b/scripts/release/configuration.py @@ -185,9 +185,12 @@ def admission_runtime(payload: Path, data: dict) -> tuple[dict, dict]: admission = read_json(payload / "deploy/judge.admission.example.json", 16384) root = read_json(payload / "deploy/sandbox.managed.runtime.example.json", 16384) for cap in admission["multi_runtime"]["capabilities"]: - image = {"c17": "gnu", "cpp20": "gnu", "python313": "python313", "java21": "java21"}[ - cap["language"] - ] + image = { + "c17": "gnu", + "cpp20": "gnu", + "python313": "python313", + "java21": "java21", + }[cap["language"]] cap["toolchain"] = data["images"][image] admission = AdmissionRegistry.model_validate(admission).model_dump(mode="json") state = "/var/lib/kf-judge/images/" + data["release_id"] @@ -234,7 +237,10 @@ def compose( if node["measurement_nonce"] is None: raise DeliveryError("explicit fresh measurement nonce required for attestation") epoch = hashlib.sha256(canonical(measurement)).hexdigest() - worker_token, platform_token = service_tokens or (secrets.token_hex(32), secrets.token_hex(32)) + worker_token, platform_token = service_tokens or ( + secrets.token_hex(32), + secrets.token_hex(32), + ) coordinator = { "listen": "127.0.0.1:8100", "database_url": endpoints["database"]["coordinator"], @@ -289,6 +295,7 @@ def compose( **common, "KF_OJ_DATABASE_URL": backend_url("import"), "KF_OJ_IMPORT_WORKER_CGROUP": IMPORT_CGROUP, + "KF_OJ_JUDGE_ADMISSION": canonical(admission).decode().strip(), } ), "identity.env": environmental( @@ -491,7 +498,12 @@ def validate_contents(paths: dict[str, Path], payload: Path, data: dict, *, atte if worker_token == platform_token: raise DeliveryError("service identities overlap") expected = compose( - payload, data, node, endpoints, attest=attest, service_tokens=(worker_token, platform_token) + payload, + data, + node, + endpoints, + attest=attest, + service_tokens=(worker_token, platform_token), ) for name in CONFIG_FILES: # JSON comparisons via canonical bytes distinguish true/1 and null/[]; @@ -523,7 +535,10 @@ def check_installed(receipt_path: Path) -> dict: raise DeliveryError("installed configuration fingerprint differs") paths[name] = path validate_contents( - paths, release / "payload", data, attest=receipt["operator_attested_measurement"] + paths, + release / "payload", + data, + attest=receipt["operator_attested_measurement"], ) return receipt diff --git a/scripts/release/service_bootstrap.py b/scripts/release/service_bootstrap.py index cb5c9d4..cdce2fc 100644 --- a/scripts/release/service_bootstrap.py +++ b/scripts/release/service_bootstrap.py @@ -1,19 +1,194 @@ -"""Ordinary Rust service launch with verified private systemd credential bytes.""" +"""Fixed ordinary service launch after effective identity and resource checks.""" import argparse import hashlib import os import re +import resource import stat from pathlib import Path from . import preflight from .policy import DeliveryError, sha256 +from .services import PROFILES ROLES = { - "coordinator": (1201, "kf-coordinator.service", "coordinator.json", "kf-coordinator"), + "coordinator": ( + 1201, + "kf-coordinator.service", + "coordinator.json", + "kf-coordinator", + ), "worker": (1202, "kf-worker.service", "worker.json", "kf-worker"), } +PYTHON_MODULES = { + "api": [ + "uvicorn", + "kf_oj.api.app:create_app", + "--factory", + "--host", + "127.0.0.1", + "--port", + "8000", + "--no-proxy-headers", + "--limit-concurrency", + "32", + "--backlog", + "64", + ], + "outbox": ["kf_oj.infra.messaging.publisher"], + "problem-import": ["kf_oj.workers.problem_import"], +} +COMMON_ENVIRONMENT = { + "KF_OJ_ENVIRONMENT", + "KF_OJ_LOG_LEVEL", + "KF_OJ_BROWSER_ORIGINS", + "KF_OJ_ARTIFACT_ROOT", + "KF_OJ_IMPORT_STAGING_ROOT", + "KF_OJ_ARTIFACT_MAX_BYTES", + "KF_OJ_ARTIFACT_MAX_FILES", + "KF_OJ_DATABASE_URL", +} +ROLE_ENVIRONMENT = { + "api": COMMON_ENVIRONMENT + | { + "KF_OJ_JUDGE_ADMISSION", + "KF_OJ_COORDINATOR_URL", + "KF_OJ_COORDINATOR_TOKEN", + }, + "outbox": COMMON_ENVIRONMENT | {"KF_OJ_AMQP_URL"}, + "problem-import": COMMON_ENVIRONMENT | {"KF_OJ_IMPORT_WORKER_CGROUP", "KF_OJ_JUDGE_ADMISSION"}, +} + + +def bounded_text(path: Path) -> str: + with path.open() as stream: + contents = stream.read(65537) + if len(contents) > 65536: + raise DeliveryError("process observation exceeds bound") + return contents + + +def observe_runtime(role: str) -> dict: + group = preflight.current_cgroup() + status = dict( + line.split(":", 1) + for line in bounded_text(Path("/proc/self/status")).splitlines() + if ":" in line + ) + limits = {} + for name, identifier in { + "core": resource.RLIMIT_CORE, + "file": resource.RLIMIT_FSIZE, + "files": resource.RLIMIT_NOFILE, + "stack": resource.RLIMIT_STACK, + }.items(): + limits[name] = list(resource.getrlimit(identifier)) + + def read_limits(path): + return { + name: bounded_text(path / name).strip() + for name in ("memory.max", "memory.swap.max", "pids.max", "cpu.max") + } + + interfaces = ( + [ + line.partition(":")[0].strip() + for line in bounded_text(Path("/proc/net/dev")).splitlines() + if ":" in line + ] + if role == "problem-import" + else None + ) + return { + "cgroup": str(group), + "service_limits": read_limits(group), + "node_limits": read_limits(Path("/sys/fs/cgroup/kfonlinejudge.slice")), + "uids": status.get("Uid", "").split(), + "gids": status.get("Gid", "").split(), + "groups": status.get("Groups", "").split(), + "no_new_privileges": status.get("NoNewPrivs", "").strip(), + "capabilities": [ + status.get(key, "").strip() + for key in ( + "CapInh", + "CapPrm", + "CapEff", + "CapBnd", + "CapAmb", + ) + ], + "pid": os.getpid(), + "process_limits": limits, + "interfaces": interfaces, + } + + +def validate_runtime(role: str, observed: dict) -> None: + if role not in PROFILES: + raise DeliveryError("unknown ordinary service") + user, memory, pids, _, _, _ = PROFILES[role] + uid = str(preflight.ACCOUNTS[user]) + if ( + observed["cgroup"] != "/sys/fs/cgroup/kfonlinejudge.slice/kf-" + role + ".service" + or observed["uids"] != [uid] * 4 + or observed["gids"] != [uid] * 4 + or not set(observed["groups"]) <= {uid} + or observed["no_new_privileges"] != "1" + or len(observed["capabilities"]) != 5 + or any(not re.fullmatch(r"0+", value) for value in observed["capabilities"]) + ): + raise DeliveryError("ordinary process identity or privilege boundary differs") + for limits, required_memory, required_pids, cpus in ( + (observed["service_limits"], memory, pids, 1), + (observed["node_limits"], 3840, 384, 2), + ): + if ( + limits["memory.max"] != str(required_memory * 1024**2) + or limits["memory.swap.max"] != "0" + or limits["pids.max"] != str(required_pids) + ): + raise DeliveryError("effective ordinary/node memory or process budget differs") + parts = limits["cpu.max"].split() + if len(parts) != 2 or not all(v.isdecimal() for v in parts): + raise DeliveryError("effective CPU budget unavailable") + quota, period = map(int, parts) + if not period or quota != cpus * period: + raise DeliveryError("effective CPU budget differs") + if observed["process_limits"] != { + "core": [0, 0], + "file": [64 * 1024**2] * 2, + "files": [64 if role == "problem-import" else 256] * 2, + "stack": [16 * 1024**2] * 2, + }: + raise DeliveryError("effective ordinary file/descriptor/stack limits differ") + if role in PYTHON_MODULES and observed["pid"] != 1: + raise DeliveryError("private ordinary PID namespace unavailable") + if role == "problem-import" and observed["interfaces"] != ["lo"]: + raise DeliveryError("private import network namespace unavailable") + + +def python_command(role: str, release: str) -> tuple[list[str], dict]: + payload = Path("/opt/kf-onlinejudge/releases") / release / "payload" + python = payload / "python/bin/python3.13" + preflight.protected(python) + environment = {key: value for key, value in os.environ.items() if key.startswith("KF_OJ_")} + if set(environment) != ROLE_ENVIRONMENT[role] or any( + not value or len(value) > 32768 or any(ch in value for ch in "\r\n\x00") + for value in environment.values() + ): + raise DeliveryError("ordinary service environment fields differ") + environment.update( + { + "PATH": "/usr/bin:/bin", + "LC_ALL": "C.UTF-8", + "PYTHONHOME": str(payload / "python"), + "PYTHONPATH": f"{payload}/backend/src:{payload}/site:{payload}/delivery", + "PYTHONDONTWRITEBYTECODE": "1", + "LD_LIBRARY_PATH": f"{payload}/lib:{payload}/python/lib", + } + ) + return [str(python), "-s", "-P", "-m", *PYTHON_MODULES[role]], environment def read_credential(path: Path, expected: str, uid: int) -> bytes: @@ -72,35 +247,53 @@ def prepare(role: str, release: str, batch: str, directory: Path, expected: str) os.close(fd) executable = Path("/opt/kf-onlinejudge/releases") / release / "payload/bin" / binary preflight.protected(executable) - return [str(executable), *(["serve"] if role == "worker" else []), "--config", str(path)] + return [ + str(executable), + *(["serve"] if role == "worker" else []), + "--config", + str(path), + ] def main() -> None: parser = argparse.ArgumentParser(description="Ordinary service credential bootstrap") - parser.add_argument("role", choices=ROLES) + parser.add_argument("role", choices=PROFILES) parser.add_argument("--release-id", required=True) parser.add_argument("--configuration-id", required=True) - parser.add_argument("--credential-directory", required=True, type=Path) - parser.add_argument("--sha256", required=True) + parser.add_argument("--credential-directory", type=Path) + parser.add_argument("--sha256") args = parser.parse_args() try: - command = prepare( - args.role, - args.release_id, - args.configuration_id, - args.credential_directory, - args.sha256, - ) + if not re.fullmatch(r"sd-[0-9a-f]{12}", args.release_id) or not re.fullmatch( + r"cfg-[0-9a-f]{24}", args.configuration_id + ): + raise DeliveryError("invalid ordinary service binding") + validate_runtime(args.role, observe_runtime(args.role)) + if args.role in PYTHON_MODULES: + if args.credential_directory is not None or args.sha256 is not None: + raise DeliveryError("unexpected ordinary credential override") + command, environment = python_command(args.role, args.release_id) + else: + if args.credential_directory is None or args.sha256 is None: + raise DeliveryError("fixed ordinary credential required") + command = prepare( + args.role, + args.release_id, + args.configuration_id, + args.credential_directory, + args.sha256, + ) + environment = { + "PATH": "/usr/bin:/bin", + "LC_ALL": "C.UTF-8", + "LD_LIBRARY_PATH": str(Path(command[0]).parent.parent / "lib"), + } # Only the verified normal service binary is launched. No student # executable or privileged Sandbox operation is accepted here. os.execve( command[0], command, - { - "PATH": "/usr/bin:/bin", - "LC_ALL": "C.UTF-8", - "LD_LIBRARY_PATH": str(Path(command[0]).parent.parent / "lib"), - }, + environment, ) except (OSError, ValueError) as error: parser.exit( diff --git a/scripts/release/services.py b/scripts/release/services.py index 2ff1cba..5937b27 100644 --- a/scripts/release/services.py +++ b/scripts/release/services.py @@ -110,13 +110,14 @@ def render(receipt: dict) -> dict[str, bytes]: "UnsetEnvironment=KF_OJ_DATABASE_URL KF_OJ_IDENTITY_OPERATOR_DATABASE_URL" " KF_OJ_COORDINATOR_TOKEN KF_OJ_AMQP_URL", ] - elif name == "api": - values += ["ExecStart=" + str(payload / "bin/kf-api")] - elif name == "outbox": - # The original scaffold console module is not a publisher loop. - values += ["ExecStart=" + python + " -s -P -m kf_oj.infra.messaging.publisher"] else: - values += ["ExecStart=" + str(payload / "bin/kf-problem-import-worker")] + values += [ + "ExecStart=" + + python + + " -s -P -m scripts.release.service_bootstrap " + + name + + f" --release-id {release} --configuration-id {batch}" + ] if name in {"api", "problem-import"}: values += [ "ReadWritePaths=/var/lib/kf-onlinejudge/artifacts /var/lib/kf-onlinejudge/imports"