feat: compose bounded ordinary service units with private credentials

backend/tests/test_delivery.py

@@ -16,6 +16,8 @@ builder = importlib.import_module("scripts.release.build")
 configuration = importlib.import_module("scripts.release.configuration")
 architecture = importlib.import_module("scripts.check_architecture")
 proxy = importlib.import_module("scripts.release.proxy")
+services = importlib.import_module("scripts.release.services")
+service_bootstrap = importlib.import_module("scripts.release.service_bootstrap")
 
 
 def test_delivery_authorities_and_layout():
@@ -330,3 +332,56 @@ def test_proxy_is_separate_from_private_execution_control():
     assert "proxy_request_buffering off" in contents
     assert "$request_uri" not in contents and "$http_cookie" not in contents
     assert value["key_sha256"] not in contents
+
+
+def test_ordinary_units_bind_exact_batch_without_inline_secrets():
+    source, checksum = "123456789abc" + "d" * 28, "f" * 64
+    hashes = {name: "b" * 64 for name in configuration.CONFIG_FILES}
+    receipt = {
+        "schema_version": 1,
+        "source_sha": source,
+        "release_id": "sd-123456789abc",
+        "configuration_id": configuration.configuration_id(source, checksum, hashes),
+        "manifest_sha256": checksum,
+        "configuration_files": hashes,
+        "operator_attested_measurement": False,
+        "installed": False,
+        "execution_verified": False,
+    }
+    units = services.render(receipt)
+    assert set(units) == {
+        "kf-api.service",
+        "kf-outbox.service",
+        "kf-coordinator.service",
+        "kf-worker.service",
+        "kf-problem-import.service",
+    }
+    for contents in units.values():
+        text = contents.decode()
+        assert "User=root" not in text
+        assert "MemorySwapMax=0" in text and "StartLimitBurst=3" in text
+        assert receipt["configuration_id"] in text and receipt["release_id"] in text
+        assert "Password=" not in text and "postgresql://" not in text and "amqp://" not in text
+    assert "PrivateNetwork=yes" in units["kf-problem-import.service"].decode()
+    assert "PrivatePIDs=yes" in units["kf-problem-import.service"].decode()
+    assert "RestrictAddressFamilies=AF_UNIX\n" in units["kf-problem-import.service"].decode()
+    assert "-m kf_oj.infra.messaging.publisher" in units["kf-outbox.service"].decode()
+    assert "LoadCredential=worker.json:" in units["kf-worker.service"].decode()
+
+
+def test_systemd_credential_bytes_are_checked_before_private_copy(tmp_path):
+    import hashlib
+    import os
+
+    path = tmp_path / "configuration.json"
+    content = b'{"synthetic_configuration":true}\n'
+    path.write_bytes(content)
+    path.chmod(0o400)
+    expected = hashlib.sha256(content).hexdigest()
+    assert service_bootstrap.read_credential(path, expected, os.getuid()) == content
+    with pytest.raises(policy.DeliveryError, match="fingerprint mismatch"):
+        service_bootstrap.read_credential(path, "b" * 64, os.getuid())
+    path.chmod(0o600)
+    path.write_bytes(b"x" * 32769)
+    with pytest.raises(policy.DeliveryError, match="bounded systemd credential"):
+        service_bootstrap.read_credential(path, expected, os.getuid())

scripts/release/README.md

@@ -107,3 +107,23 @@ silently substitutes a healthy result when lifecycle evidence is absent.
 Proxy directive references: [NGINX core](https://nginx.org/en/docs/http/ngx_http_core_module.html),
 [proxy module](https://nginx.org/en/docs/http/ngx_http_proxy_module.html), and
 [TLS module](https://nginx.org/en/docs/http/ngx_http_ssl_module.html).
+
+`python3.13 -m scripts.release.services generate` composes five ordinary system
+units from a verified bundle and matching private configuration receipt. The
+matching service launch modules must themselves be in that bundle. Rendering
+uses fixed users, resource ceilings, finite restart/stop limits and an explicit
+release/configuration pre-start binding. It never writes system units, starts a
+process, migrates a schema or claims readiness. Actual units, mount/PID/cgroup
+views, Unix DB socket access, dependency roles and effective limits require VM
+installation/acceptance. Root factory and database/broker lifecycle are separate.
+
+API/outbox/import share the established content-storage identity; individual PID
+namespaces prevent same-UID inspection of another service's environment. This
+requires the fixed systemd profile and verified kernel namespace support. Rust
+Coordinator/Worker use distinct ordinary identities and systemd credentials.
+`service_bootstrap` verifies those bounded credential bytes, copies them to a
+private service runtime directory with the existing required 0600 mode and then
+launches only the fixed ordinary binary. It accepts no student executable,
+command override or privileged Sandbox operation. Tokens/URLs are not command
+arguments or unit text. The outbox unit uses the existing actual confirmed
+publisher module; the old scaffold check entry is not treated as a delivery loop.

scripts/release/service_bootstrap.py

@@ -0,0 +1,115 @@
+"""Ordinary Rust service launch with verified private systemd credential bytes."""
+
+import argparse
+import hashlib
+import os
+import re
+import stat
+from pathlib import Path
+
+from . import preflight
+from .policy import DeliveryError, sha256
+
+ROLES = {
+    "coordinator": (1201, "kf-coordinator.service", "coordinator.json", "kf-coordinator"),
+    "worker": (1202, "kf-worker.service", "worker.json", "kf-worker"),
+}
+
+
+def read_credential(path: Path, expected: str, uid: int) -> bytes:
+    info = path.lstat()
+    if (
+        not stat.S_ISREG(info.st_mode)
+        or info.st_nlink != 1
+        or info.st_uid not in {0, uid}
+        or stat.S_IMODE(info.st_mode) not in {0o400, 0o600}
+        or not 1 <= info.st_size <= 32768
+    ):
+        raise DeliveryError("private bounded systemd credential required")
+    content = path.read_bytes()
+    if len(content) != info.st_size or hashlib.sha256(content).hexdigest() != sha256(expected):
+        raise DeliveryError("service credential fingerprint mismatch")
+    return content
+
+
+def prepare(role: str, release: str, batch: str, directory: Path, expected: str) -> list[str]:
+    if (
+        role not in ROLES
+        or not re.fullmatch(r"sd-[0-9a-f]{12}", release)
+        or not re.fullmatch(r"cfg-[0-9a-f]{24}", batch)
+    ):
+        raise DeliveryError("invalid ordinary service binding")
+    uid, unit, member, binary = ROLES[role]
+    if os.getuid() != uid or os.geteuid() != uid or os.getgid() != uid:
+        raise DeliveryError("dedicated ordinary service identity required")
+    if directory != Path("/run/credentials") / unit or directory.is_symlink():
+        raise DeliveryError("fixed systemd credential directory required")
+    if preflight.current_cgroup() != Path("/sys/fs/cgroup/kfonlinejudge.slice") / unit:
+        raise DeliveryError("service cgroup identity differs")
+    content = read_credential(directory / member, expected, uid)
+    runtime = Path("/run") / ("kf-" + role)
+    info = runtime.lstat()
+    if not stat.S_ISDIR(info.st_mode) or info.st_uid != uid or stat.S_IMODE(info.st_mode) != 0o700:
+        raise DeliveryError("private ordinary runtime directory required")
+    path = runtime / "configuration.json"
+    if path.exists() or path.is_symlink():
+        old = path.lstat()
+        if (
+            not stat.S_ISREG(old.st_mode)
+            or old.st_nlink != 1
+            or old.st_uid != uid
+            or (stat.S_IMODE(old.st_mode) != 0o600 or path.read_bytes() != content)
+        ):
+            raise DeliveryError("unknown runtime configuration refused")
+    else:
+        fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600)
+        try:
+            with os.fdopen(fd, "wb", closefd=False) as target:
+                target.write(content)
+                target.flush()
+                os.fsync(fd)
+        finally:
+            os.close(fd)
+    executable = Path("/opt/kf-onlinejudge/releases") / release / "payload/bin" / binary
+    preflight.protected(executable)
+    return [str(executable), *(["serve"] if role == "worker" else []), "--config", str(path)]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Ordinary service credential bootstrap")
+    parser.add_argument("role", choices=ROLES)
+    parser.add_argument("--release-id", required=True)
+    parser.add_argument("--configuration-id", required=True)
+    parser.add_argument("--credential-directory", required=True, type=Path)
+    parser.add_argument("--sha256", required=True)
+    args = parser.parse_args()
+    try:
+        command = prepare(
+            args.role,
+            args.release_id,
+            args.configuration_id,
+            args.credential_directory,
+            args.sha256,
+        )
+        # Only the verified normal service binary is launched. No student
+        # executable or privileged Sandbox operation is accepted here.
+        os.execve(
+            command[0],
+            command,
+            {
+                "PATH": "/usr/bin:/bin",
+                "LC_ALL": "C.UTF-8",
+                "LD_LIBRARY_PATH": str(Path(command[0]).parent.parent / "lib"),
+            },
+        )
+    except (OSError, ValueError) as error:
+        parser.exit(
+            1,
+            "Ordinary service refused: "
+            + (str(error) if isinstance(error, DeliveryError) else type(error).__name__)
+            + "\n",
+        )
+
+
+if __name__ == "__main__":
+    main()

scripts/release/services.py

@@ -0,0 +1,229 @@
+"""Ordinary service composition; rendering is not Linux service acceptance."""
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+from . import configuration, install, manifest
+from .policy import DeliveryError, canonical, digest
+
+PROFILES = {
+    "api": ("kf-oj", 256, 32, 100, "api.env", None),
+    "coordinator": ("kf-coordinator", 128, 16, 100, None, "coordinator.json"),
+    "outbox": ("kf-oj", 128, 16, 100, "outbox.env", None),
+    "problem-import": ("kf-oj", 128, 16, 100, "import.env", None),
+    "worker": ("kf-worker", 128, 32, 100, None, "worker.json"),
+}
+
+
+def python_environment(payload: Path) -> str:
+    return (
+        f"PYTHONHOME={payload}/python "
+        f"PYTHONPATH={payload}/backend/src:{payload}/site:{payload}/delivery "
+        f"LD_LIBRARY_PATH={payload}/lib:{payload}/python/lib PYTHONDONTWRITEBYTECODE=1"
+    )
+
+
+def render(receipt: dict) -> dict[str, bytes]:
+    receipt = configuration.validate_receipt(receipt, installed=False)
+    release, batch = receipt["release_id"], receipt["configuration_id"]
+    payload = install.RELEASES / release / "payload"
+    python = str(payload / "python/bin/python3.13")
+    files = {}
+    for name, (user, memory, pids, cpu, environment, credential) in PROFILES.items():
+        unit_name = "kf-" + name + ".service"
+        runtime = "kf-" + name
+        values = [
+            "[Unit]",
+            "Description=KF bounded ordinary " + name,
+            "After=network.target kf-postgres.service kf-rabbitmq.service",
+            "StartLimitIntervalSec=600",
+            "StartLimitBurst=3",
+            "[Service]",
+            "Type=exec",
+            "Slice=kfonlinejudge.slice",
+            "User=" + user,
+            "Group=" + user,
+            "WorkingDirectory=" + str(payload),
+            "Environment=PATH=/usr/bin:/bin LC_ALL=C.UTF-8 " + python_environment(payload),
+            "RuntimeDirectory=" + runtime,
+            "RuntimeDirectoryMode=0700",
+            "UMask=0077",
+            "ExecStartPre=+/usr/bin/env -i PATH=/usr/bin:/bin LC_ALL=C "
+            + python_environment(payload)
+            + f" {python} -s -P -m scripts.release.services check-binding"
+            + f" --release-id {release} --configuration-id {batch}",
+            f"MemoryMax={memory}M",
+            "MemorySwapMax=0",
+            f"TasksMax={pids}",
+            f"CPUQuota={cpu}%",
+            "LimitCORE=0",
+            "LimitFSIZE=64M",
+            "LimitNOFILE=" + ("64" if name == "problem-import" else "256"),
+            "LimitSTACK=16M",
+            "NoNewPrivileges=yes",
+            "CapabilityBoundingSet=",
+            "AmbientCapabilities=",
+            "ProtectSystem=strict",
+            "ProtectHome=yes",
+            "ProtectKernelTunables=yes",
+            "ProtectKernelModules=yes",
+            "ProtectControlGroups=yes",
+            "ProtectProc=invisible",
+            "RestrictNamespaces=yes",
+            "RestrictSUIDSGID=yes",
+            "LockPersonality=yes",
+            "PrivateTmp=yes",
+            "PrivateDevices=yes",
+            "KillMode=control-group",
+            "TimeoutStartSec=120",
+            "TimeoutStopSec=30",
+            "Restart=on-failure",
+            "RestartSec=30",
+            "RuntimeMaxSec=24h",
+            "StandardOutput=journal",
+            "StandardError=journal",
+            "LogRateLimitIntervalSec=30s",
+            "LogRateLimitBurst=100",
+        ]
+        if environment:
+            path = configuration.installed_path(release, batch, environment)
+            values += ["EnvironmentFile=" + str(path)]
+            # The common storage identity is intentional. A separate PID
+            # namespace prevents the import parser from reading API credentials
+            # from a same-UID process; actual namespace support is verified in VM.
+            values += ["PrivatePIDs=yes"]
+        if credential:
+            values += [
+                "LoadCredential="
+                + credential
+                + ":"
+                + str(configuration.installed_path(release, batch, credential)),
+                "ExecStart="
+                + python
+                + " -s -P -m scripts.release.service_bootstrap "
+                + name
+                + f" --release-id {release} --configuration-id {batch}"
+                + " --credential-directory=%d --sha256="
+                + receipt["configuration_files"][credential],
+                "UnsetEnvironment=KF_OJ_DATABASE_URL KF_OJ_IDENTITY_OPERATOR_DATABASE_URL"
+                " KF_OJ_COORDINATOR_TOKEN KF_OJ_AMQP_URL",
+            ]
+        elif name == "api":
+            values += ["ExecStart=" + str(payload / "bin/kf-api")]
+        elif name == "outbox":
+            # The original scaffold console module is not a publisher loop.
+            values += ["ExecStart=" + python + " -s -P -m kf_oj.infra.messaging.publisher"]
+        else:
+            values += ["ExecStart=" + str(payload / "bin/kf-problem-import-worker")]
+        if name in {"api", "problem-import"}:
+            values += [
+                "ReadWritePaths=/var/lib/kf-onlinejudge/artifacts /var/lib/kf-onlinejudge/imports"
+            ]
+        elif name == "worker":
+            values += ["ReadWritePaths=/var/lib/kf-onlinejudge/worker"]
+        if name == "problem-import":
+            values += ["PrivateNetwork=yes", "RestrictAddressFamilies=AF_UNIX"]
+        else:
+            values += ["RestrictAddressFamilies=AF_UNIX AF_INET"]
+        values += ["[Install]", "WantedBy=multi-user.target"]
+        files[unit_name] = ("\n".join(values) + "\n").encode()
+    return files
+
+
+def generate(
+    bundle: Path, expected: str, source: Path, expected_receipt: str, output: Path
+) -> dict:
+    data = manifest.verify(bundle, expected)
+    for name in ("services.py", "service_bootstrap.py"):
+        shipped = bundle / "payload/delivery/scripts/release" / name
+        if not shipped.is_file() or digest(shipped) != digest(Path(__file__).parent / name):
+            raise DeliveryError("bundle lacks the matching ordinary service launch modules")
+    receipt_file = source / "receipt.json"
+    configuration.private(receipt_file)
+    if digest(receipt_file) != expected_receipt:
+        raise DeliveryError("service source configuration receipt differs")
+    from .policy import read_json
+
+    receipt = configuration.validate_receipt(read_json(receipt_file, 16384), installed=False)
+    if receipt["source_sha"] != data["source_sha"] or receipt["manifest_sha256"] != expected:
+        raise DeliveryError("service configuration source differs from bundle")
+    if {p.name for p in source.iterdir()} != configuration.CONFIG_FILES | {"receipt.json"}:
+        raise DeliveryError("unexpected service configuration member")
+    paths = {name: source / name for name in configuration.CONFIG_FILES}
+    for name, path in paths.items():
+        configuration.private(path)
+        if digest(path) != receipt["configuration_files"][name]:
+            raise DeliveryError("service configuration bytes differ")
+    configuration.validate_contents(
+        paths, bundle / "payload", data, attest=receipt["operator_attested_measurement"]
+    )
+    if output.exists() or output.is_symlink():
+        raise DeliveryError("service output must be new")
+    output.mkdir(mode=0o700)
+    fingerprints = {}
+    for name, contents in render(receipt).items():
+        path = output / name
+        path.write_bytes(contents)
+        path.chmod(0o600)
+        fingerprints[name] = digest(path)
+    result = {
+        "schema_version": 1,
+        "source_sha": data["source_sha"],
+        "configuration_id": receipt["configuration_id"],
+        "configuration_receipt_sha256": expected_receipt,
+        "files": fingerprints,
+        "installed": False,
+        "services_started": False,
+        "execution_verified": False,
+    }
+    (output / "receipt.json").write_bytes(canonical(result))
+    (output / "receipt.json").chmod(0o600)
+    return result
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Bounded ordinary service composition")
+    sub = parser.add_subparsers(dest="command", required=True)
+    check = sub.add_parser("check-binding")
+    check.add_argument("--release-id", required=True)
+    check.add_argument("--configuration-id", required=True)
+    check = sub.add_parser("generate")
+    check.add_argument("bundle", type=Path)
+    check.add_argument("--manifest-sha256", required=True)
+    check.add_argument("--configuration", type=Path, required=True)
+    check.add_argument("--configuration-receipt-sha256", required=True)
+    check.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    try:
+        if args.command == "check-binding":
+            if os.geteuid() != 0:
+                raise DeliveryError("protected service binding check requires operator")
+            active = install.selection()
+            if (active["release_id"], active["configuration_id"]) != (
+                args.release_id,
+                args.configuration_id,
+            ):
+                raise DeliveryError("unit belongs to an unselected release/configuration")
+            result = {"binding_valid": True, "execution_verified": False}
+        else:
+            result = generate(
+                args.bundle,
+                args.manifest_sha256,
+                args.configuration,
+                args.configuration_receipt_sha256,
+                args.output,
+            )
+        print(json.dumps(result, sort_keys=True))
+    except (OSError, ValueError) as error:
+        parser.exit(
+            1,
+            "Service refused: "
+            + (str(error) if isinstance(error, DeliveryError) else type(error).__name__)
+            + "\n",
+        )
+
+
+if __name__ == "__main__":
+    main()