diff --git a/backend/tests/test_delivery.py b/backend/tests/test_delivery.py index f3368ec..1ae7f60 100644 --- a/backend/tests/test_delivery.py +++ b/backend/tests/test_delivery.py @@ -16,6 +16,8 @@ builder = importlib.import_module("scripts.release.build") configuration = importlib.import_module("scripts.release.configuration") architecture = importlib.import_module("scripts.check_architecture") proxy = importlib.import_module("scripts.release.proxy") +services = importlib.import_module("scripts.release.services") +service_bootstrap = importlib.import_module("scripts.release.service_bootstrap") def test_delivery_authorities_and_layout(): @@ -330,3 +332,56 @@ def test_proxy_is_separate_from_private_execution_control(): assert "proxy_request_buffering off" in contents assert "$request_uri" not in contents and "$http_cookie" not in contents assert value["key_sha256"] not in contents + + +def test_ordinary_units_bind_exact_batch_without_inline_secrets(): + source, checksum = "123456789abc" + "d" * 28, "f" * 64 + hashes = {name: "b" * 64 for name in configuration.CONFIG_FILES} + receipt = { + "schema_version": 1, + "source_sha": source, + "release_id": "sd-123456789abc", + "configuration_id": configuration.configuration_id(source, checksum, hashes), + "manifest_sha256": checksum, + "configuration_files": hashes, + "operator_attested_measurement": False, + "installed": False, + "execution_verified": False, + } + units = services.render(receipt) + assert set(units) == { + "kf-api.service", + "kf-outbox.service", + "kf-coordinator.service", + "kf-worker.service", + "kf-problem-import.service", + } + for contents in units.values(): + text = contents.decode() + assert "User=root" not in text + assert "MemorySwapMax=0" in text and "StartLimitBurst=3" in text + assert receipt["configuration_id"] in text and receipt["release_id"] in text + assert "Password=" not in text and "postgresql://" not in text and "amqp://" not in text + assert "PrivateNetwork=yes" in units["kf-problem-import.service"].decode() + assert "PrivatePIDs=yes" in units["kf-problem-import.service"].decode() + assert "RestrictAddressFamilies=AF_UNIX\n" in units["kf-problem-import.service"].decode() + assert "-m kf_oj.infra.messaging.publisher" in units["kf-outbox.service"].decode() + assert "LoadCredential=worker.json:" in units["kf-worker.service"].decode() + + +def test_systemd_credential_bytes_are_checked_before_private_copy(tmp_path): + import hashlib + import os + + path = tmp_path / "configuration.json" + content = b'{"synthetic_configuration":true}\n' + path.write_bytes(content) + path.chmod(0o400) + expected = hashlib.sha256(content).hexdigest() + assert service_bootstrap.read_credential(path, expected, os.getuid()) == content + with pytest.raises(policy.DeliveryError, match="fingerprint mismatch"): + service_bootstrap.read_credential(path, "b" * 64, os.getuid()) + path.chmod(0o600) + path.write_bytes(b"x" * 32769) + with pytest.raises(policy.DeliveryError, match="bounded systemd credential"): + service_bootstrap.read_credential(path, expected, os.getuid()) diff --git a/scripts/release/README.md b/scripts/release/README.md index 53d02ca..d3065f1 100644 --- a/scripts/release/README.md +++ b/scripts/release/README.md @@ -107,3 +107,23 @@ silently substitutes a healthy result when lifecycle evidence is absent. Proxy directive references: [NGINX core](https://nginx.org/en/docs/http/ngx_http_core_module.html), [proxy module](https://nginx.org/en/docs/http/ngx_http_proxy_module.html), and [TLS module](https://nginx.org/en/docs/http/ngx_http_ssl_module.html). + +`python3.13 -m scripts.release.services generate` composes five ordinary system +units from a verified bundle and matching private configuration receipt. The +matching service launch modules must themselves be in that bundle. Rendering +uses fixed users, resource ceilings, finite restart/stop limits and an explicit +release/configuration pre-start binding. It never writes system units, starts a +process, migrates a schema or claims readiness. Actual units, mount/PID/cgroup +views, Unix DB socket access, dependency roles and effective limits require VM +installation/acceptance. Root factory and database/broker lifecycle are separate. + +API/outbox/import share the established content-storage identity; individual PID +namespaces prevent same-UID inspection of another service's environment. This +requires the fixed systemd profile and verified kernel namespace support. Rust +Coordinator/Worker use distinct ordinary identities and systemd credentials. +`service_bootstrap` verifies those bounded credential bytes, copies them to a +private service runtime directory with the existing required 0600 mode and then +launches only the fixed ordinary binary. It accepts no student executable, +command override or privileged Sandbox operation. Tokens/URLs are not command +arguments or unit text. The outbox unit uses the existing actual confirmed +publisher module; the old scaffold check entry is not treated as a delivery loop. diff --git a/scripts/release/service_bootstrap.py b/scripts/release/service_bootstrap.py new file mode 100644 index 0000000..cb5c9d4 --- /dev/null +++ b/scripts/release/service_bootstrap.py @@ -0,0 +1,115 @@ +"""Ordinary Rust service launch with verified private systemd credential bytes.""" + +import argparse +import hashlib +import os +import re +import stat +from pathlib import Path + +from . import preflight +from .policy import DeliveryError, sha256 + +ROLES = { + "coordinator": (1201, "kf-coordinator.service", "coordinator.json", "kf-coordinator"), + "worker": (1202, "kf-worker.service", "worker.json", "kf-worker"), +} + + +def read_credential(path: Path, expected: str, uid: int) -> bytes: + info = path.lstat() + if ( + not stat.S_ISREG(info.st_mode) + or info.st_nlink != 1 + or info.st_uid not in {0, uid} + or stat.S_IMODE(info.st_mode) not in {0o400, 0o600} + or not 1 <= info.st_size <= 32768 + ): + raise DeliveryError("private bounded systemd credential required") + content = path.read_bytes() + if len(content) != info.st_size or hashlib.sha256(content).hexdigest() != sha256(expected): + raise DeliveryError("service credential fingerprint mismatch") + return content + + +def prepare(role: str, release: str, batch: str, directory: Path, expected: str) -> list[str]: + if ( + role not in ROLES + or not re.fullmatch(r"sd-[0-9a-f]{12}", release) + or not re.fullmatch(r"cfg-[0-9a-f]{24}", batch) + ): + raise DeliveryError("invalid ordinary service binding") + uid, unit, member, binary = ROLES[role] + if os.getuid() != uid or os.geteuid() != uid or os.getgid() != uid: + raise DeliveryError("dedicated ordinary service identity required") + if directory != Path("/run/credentials") / unit or directory.is_symlink(): + raise DeliveryError("fixed systemd credential directory required") + if preflight.current_cgroup() != Path("/sys/fs/cgroup/kfonlinejudge.slice") / unit: + raise DeliveryError("service cgroup identity differs") + content = read_credential(directory / member, expected, uid) + runtime = Path("/run") / ("kf-" + role) + info = runtime.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != uid or stat.S_IMODE(info.st_mode) != 0o700: + raise DeliveryError("private ordinary runtime directory required") + path = runtime / "configuration.json" + if path.exists() or path.is_symlink(): + old = path.lstat() + if ( + not stat.S_ISREG(old.st_mode) + or old.st_nlink != 1 + or old.st_uid != uid + or (stat.S_IMODE(old.st_mode) != 0o600 or path.read_bytes() != content) + ): + raise DeliveryError("unknown runtime configuration refused") + else: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + try: + with os.fdopen(fd, "wb", closefd=False) as target: + target.write(content) + target.flush() + os.fsync(fd) + finally: + os.close(fd) + executable = Path("/opt/kf-onlinejudge/releases") / release / "payload/bin" / binary + preflight.protected(executable) + return [str(executable), *(["serve"] if role == "worker" else []), "--config", str(path)] + + +def main() -> None: + parser = argparse.ArgumentParser(description="Ordinary service credential bootstrap") + parser.add_argument("role", choices=ROLES) + parser.add_argument("--release-id", required=True) + parser.add_argument("--configuration-id", required=True) + parser.add_argument("--credential-directory", required=True, type=Path) + parser.add_argument("--sha256", required=True) + args = parser.parse_args() + try: + command = prepare( + args.role, + args.release_id, + args.configuration_id, + args.credential_directory, + args.sha256, + ) + # Only the verified normal service binary is launched. No student + # executable or privileged Sandbox operation is accepted here. + os.execve( + command[0], + command, + { + "PATH": "/usr/bin:/bin", + "LC_ALL": "C.UTF-8", + "LD_LIBRARY_PATH": str(Path(command[0]).parent.parent / "lib"), + }, + ) + except (OSError, ValueError) as error: + parser.exit( + 1, + "Ordinary service refused: " + + (str(error) if isinstance(error, DeliveryError) else type(error).__name__) + + "\n", + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/services.py b/scripts/release/services.py new file mode 100644 index 0000000..2ff1cba --- /dev/null +++ b/scripts/release/services.py @@ -0,0 +1,229 @@ +"""Ordinary service composition; rendering is not Linux service acceptance.""" + +import argparse +import json +import os +from pathlib import Path + +from . import configuration, install, manifest +from .policy import DeliveryError, canonical, digest + +PROFILES = { + "api": ("kf-oj", 256, 32, 100, "api.env", None), + "coordinator": ("kf-coordinator", 128, 16, 100, None, "coordinator.json"), + "outbox": ("kf-oj", 128, 16, 100, "outbox.env", None), + "problem-import": ("kf-oj", 128, 16, 100, "import.env", None), + "worker": ("kf-worker", 128, 32, 100, None, "worker.json"), +} + + +def python_environment(payload: Path) -> str: + return ( + f"PYTHONHOME={payload}/python " + f"PYTHONPATH={payload}/backend/src:{payload}/site:{payload}/delivery " + f"LD_LIBRARY_PATH={payload}/lib:{payload}/python/lib PYTHONDONTWRITEBYTECODE=1" + ) + + +def render(receipt: dict) -> dict[str, bytes]: + receipt = configuration.validate_receipt(receipt, installed=False) + release, batch = receipt["release_id"], receipt["configuration_id"] + payload = install.RELEASES / release / "payload" + python = str(payload / "python/bin/python3.13") + files = {} + for name, (user, memory, pids, cpu, environment, credential) in PROFILES.items(): + unit_name = "kf-" + name + ".service" + runtime = "kf-" + name + values = [ + "[Unit]", + "Description=KF bounded ordinary " + name, + "After=network.target kf-postgres.service kf-rabbitmq.service", + "StartLimitIntervalSec=600", + "StartLimitBurst=3", + "[Service]", + "Type=exec", + "Slice=kfonlinejudge.slice", + "User=" + user, + "Group=" + user, + "WorkingDirectory=" + str(payload), + "Environment=PATH=/usr/bin:/bin LC_ALL=C.UTF-8 " + python_environment(payload), + "RuntimeDirectory=" + runtime, + "RuntimeDirectoryMode=0700", + "UMask=0077", + "ExecStartPre=+/usr/bin/env -i PATH=/usr/bin:/bin LC_ALL=C " + + python_environment(payload) + + f" {python} -s -P -m scripts.release.services check-binding" + + f" --release-id {release} --configuration-id {batch}", + f"MemoryMax={memory}M", + "MemorySwapMax=0", + f"TasksMax={pids}", + f"CPUQuota={cpu}%", + "LimitCORE=0", + "LimitFSIZE=64M", + "LimitNOFILE=" + ("64" if name == "problem-import" else "256"), + "LimitSTACK=16M", + "NoNewPrivileges=yes", + "CapabilityBoundingSet=", + "AmbientCapabilities=", + "ProtectSystem=strict", + "ProtectHome=yes", + "ProtectKernelTunables=yes", + "ProtectKernelModules=yes", + "ProtectControlGroups=yes", + "ProtectProc=invisible", + "RestrictNamespaces=yes", + "RestrictSUIDSGID=yes", + "LockPersonality=yes", + "PrivateTmp=yes", + "PrivateDevices=yes", + "KillMode=control-group", + "TimeoutStartSec=120", + "TimeoutStopSec=30", + "Restart=on-failure", + "RestartSec=30", + "RuntimeMaxSec=24h", + "StandardOutput=journal", + "StandardError=journal", + "LogRateLimitIntervalSec=30s", + "LogRateLimitBurst=100", + ] + if environment: + path = configuration.installed_path(release, batch, environment) + values += ["EnvironmentFile=" + str(path)] + # The common storage identity is intentional. A separate PID + # namespace prevents the import parser from reading API credentials + # from a same-UID process; actual namespace support is verified in VM. + values += ["PrivatePIDs=yes"] + if credential: + values += [ + "LoadCredential=" + + credential + + ":" + + str(configuration.installed_path(release, batch, credential)), + "ExecStart=" + + python + + " -s -P -m scripts.release.service_bootstrap " + + name + + f" --release-id {release} --configuration-id {batch}" + + " --credential-directory=%d --sha256=" + + receipt["configuration_files"][credential], + "UnsetEnvironment=KF_OJ_DATABASE_URL KF_OJ_IDENTITY_OPERATOR_DATABASE_URL" + " KF_OJ_COORDINATOR_TOKEN KF_OJ_AMQP_URL", + ] + elif name == "api": + values += ["ExecStart=" + str(payload / "bin/kf-api")] + elif name == "outbox": + # The original scaffold console module is not a publisher loop. + values += ["ExecStart=" + python + " -s -P -m kf_oj.infra.messaging.publisher"] + else: + values += ["ExecStart=" + str(payload / "bin/kf-problem-import-worker")] + if name in {"api", "problem-import"}: + values += [ + "ReadWritePaths=/var/lib/kf-onlinejudge/artifacts /var/lib/kf-onlinejudge/imports" + ] + elif name == "worker": + values += ["ReadWritePaths=/var/lib/kf-onlinejudge/worker"] + if name == "problem-import": + values += ["PrivateNetwork=yes", "RestrictAddressFamilies=AF_UNIX"] + else: + values += ["RestrictAddressFamilies=AF_UNIX AF_INET"] + values += ["[Install]", "WantedBy=multi-user.target"] + files[unit_name] = ("\n".join(values) + "\n").encode() + return files + + +def generate( + bundle: Path, expected: str, source: Path, expected_receipt: str, output: Path +) -> dict: + data = manifest.verify(bundle, expected) + for name in ("services.py", "service_bootstrap.py"): + shipped = bundle / "payload/delivery/scripts/release" / name + if not shipped.is_file() or digest(shipped) != digest(Path(__file__).parent / name): + raise DeliveryError("bundle lacks the matching ordinary service launch modules") + receipt_file = source / "receipt.json" + configuration.private(receipt_file) + if digest(receipt_file) != expected_receipt: + raise DeliveryError("service source configuration receipt differs") + from .policy import read_json + + receipt = configuration.validate_receipt(read_json(receipt_file, 16384), installed=False) + if receipt["source_sha"] != data["source_sha"] or receipt["manifest_sha256"] != expected: + raise DeliveryError("service configuration source differs from bundle") + if {p.name for p in source.iterdir()} != configuration.CONFIG_FILES | {"receipt.json"}: + raise DeliveryError("unexpected service configuration member") + paths = {name: source / name for name in configuration.CONFIG_FILES} + for name, path in paths.items(): + configuration.private(path) + if digest(path) != receipt["configuration_files"][name]: + raise DeliveryError("service configuration bytes differ") + configuration.validate_contents( + paths, bundle / "payload", data, attest=receipt["operator_attested_measurement"] + ) + if output.exists() or output.is_symlink(): + raise DeliveryError("service output must be new") + output.mkdir(mode=0o700) + fingerprints = {} + for name, contents in render(receipt).items(): + path = output / name + path.write_bytes(contents) + path.chmod(0o600) + fingerprints[name] = digest(path) + result = { + "schema_version": 1, + "source_sha": data["source_sha"], + "configuration_id": receipt["configuration_id"], + "configuration_receipt_sha256": expected_receipt, + "files": fingerprints, + "installed": False, + "services_started": False, + "execution_verified": False, + } + (output / "receipt.json").write_bytes(canonical(result)) + (output / "receipt.json").chmod(0o600) + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description="Bounded ordinary service composition") + sub = parser.add_subparsers(dest="command", required=True) + check = sub.add_parser("check-binding") + check.add_argument("--release-id", required=True) + check.add_argument("--configuration-id", required=True) + check = sub.add_parser("generate") + check.add_argument("bundle", type=Path) + check.add_argument("--manifest-sha256", required=True) + check.add_argument("--configuration", type=Path, required=True) + check.add_argument("--configuration-receipt-sha256", required=True) + check.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + if args.command == "check-binding": + if os.geteuid() != 0: + raise DeliveryError("protected service binding check requires operator") + active = install.selection() + if (active["release_id"], active["configuration_id"]) != ( + args.release_id, + args.configuration_id, + ): + raise DeliveryError("unit belongs to an unselected release/configuration") + result = {"binding_valid": True, "execution_verified": False} + else: + result = generate( + args.bundle, + args.manifest_sha256, + args.configuration, + args.configuration_receipt_sha256, + args.output, + ) + print(json.dumps(result, sort_keys=True)) + except (OSError, ValueError) as error: + parser.exit( + 1, + "Service refused: " + + (str(error) if isinstance(error, DeliveryError) else type(error).__name__) + + "\n", + ) + + +if __name__ == "__main__": + main()