Kronecker/KF-OnlineJudge
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(delivery): verify ordinary service launch boundaries

Browse Source
This commit is contained in:
Kronecker
2026-10-13 00:18:47 +08:00
parent 5c7146d245
commit 743d6ef429
5 changed files with 320 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/release/README.md
+11 -1
View File
@@ -121,9 +121,19 @@ API/outbox/import share the established content-storage identity; individual PID
namespaces prevent same-UID inspection of another service's environment. This
requires the fixed systemd profile and verified kernel namespace support. Rust
Coordinator/Worker use distinct ordinary identities and systemd credentials.
`service_bootstrap` verifies those bounded credential bytes, copies them to a
Every ordinary launch checks effective service/node cgroup memory, swap, CPU and
PID limits, file/descriptor/stack limits, UID/GID and absence of capabilities
before exec. The three Python services also require PID 1 in their private PID
namespace; import requires a loopback-only network view. Ignored namespace
directives therefore refuse startup. These checks still require actual VM
verification; they do not attest Sandbox isolation or service readiness.
The import environment includes the same validated admission policy as API.
`service_bootstrap` verifies bounded Rust credential bytes, copies them to a
private service runtime directory with the existing required 0600 mode and then
launches only the fixed ordinary binary. It accepts no student executable,
command override or privileged Sandbox operation. Tokens/URLs are not command
arguments or unit text. The outbox unit uses the existing actual confirmed
publisher module; the old scaffold check entry is not treated as a delivery loop.
Python launches use a fixed module/argument list and only the generated service
environment fields; inherited debugging/proxy/provider credentials are omitted.
scripts/release/configuration.py
+21 -6
View File
@@ -185,9 +185,12 @@ def admission_runtime(payload: Path, data: dict) -> tuple[dict, dict]:
admission = read_json(payload / "deploy/judge.admission.example.json", 16384)
root = read_json(payload / "deploy/sandbox.managed.runtime.example.json", 16384)
for cap in admission["multi_runtime"]["capabilities"]:
image = {"c17": "gnu", "cpp20": "gnu", "python313": "python313", "java21": "java21"}[
cap["language"]
]
image = {
"c17": "gnu",
"cpp20": "gnu",
"python313": "python313",
"java21": "java21",
}[cap["language"]]
cap["toolchain"] = data["images"][image]
admission = AdmissionRegistry.model_validate(admission).model_dump(mode="json")
state = "/var/lib/kf-judge/images/" + data["release_id"]
@@ -234,7 +237,10 @@ def compose(
if node["measurement_nonce"] is None:
raise DeliveryError("explicit fresh measurement nonce required for attestation")
epoch = hashlib.sha256(canonical(measurement)).hexdigest()
worker_token, platform_token = service_tokens or (secrets.token_hex(32), secrets.token_hex(32))
worker_token, platform_token = service_tokens or (
secrets.token_hex(32),
secrets.token_hex(32),
)
coordinator = {
"listen": "127.0.0.1:8100",
"database_url": endpoints["database"]["coordinator"],
@@ -289,6 +295,7 @@ def compose(
**common,
"KF_OJ_DATABASE_URL": backend_url("import"),
"KF_OJ_IMPORT_WORKER_CGROUP": IMPORT_CGROUP,
"KF_OJ_JUDGE_ADMISSION": canonical(admission).decode().strip(),
}
),
"identity.env": environmental(
@@ -491,7 +498,12 @@ def validate_contents(paths: dict[str, Path], payload: Path, data: dict, *, atte
if worker_token == platform_token:
raise DeliveryError("service identities overlap")
expected = compose(
payload, data, node, endpoints, attest=attest, service_tokens=(worker_token, platform_token)
payload,
data,
node,
endpoints,
attest=attest,
service_tokens=(worker_token, platform_token),
)
for name in CONFIG_FILES:
# JSON comparisons via canonical bytes distinguish true/1 and null/[];
@@ -523,7 +535,10 @@ def check_installed(receipt_path: Path) -> dict:
raise DeliveryError("installed configuration fingerprint differs")
paths[name] = path
validate_contents(
paths, release / "payload", data, attest=receipt["operator_attested_measurement"]
paths,
release / "payload",
data,
attest=receipt["operator_attested_measurement"],
)
return receipt
scripts/release/service_bootstrap.py
+211 -18
View File
@@ -1,19 +1,194 @@
"""Ordinary Rust service launch with verified private systemd credential bytes."""
"""Fixed ordinary service launch after effective identity and resource checks."""
import argparse
import hashlib
import os
import re
import resource
import stat
from pathlib import Path
from . import preflight
from .policy import DeliveryError, sha256
from .services import PROFILES
ROLES = {
"coordinator": (1201, "kf-coordinator.service", "coordinator.json", "kf-coordinator"),
"coordinator": (
1201,
"kf-coordinator.service",
"coordinator.json",
"kf-coordinator",
),
"worker": (1202, "kf-worker.service", "worker.json", "kf-worker"),
}
PYTHON_MODULES = {
"api": [
"uvicorn",
"kf_oj.api.app:create_app",
"--factory",
"--host",
"127.0.0.1",
"--port",
"8000",
"--no-proxy-headers",
"--limit-concurrency",
"32",
"--backlog",
"64",
],
"outbox": ["kf_oj.infra.messaging.publisher"],
"problem-import": ["kf_oj.workers.problem_import"],
}
COMMON_ENVIRONMENT = {
"KF_OJ_ENVIRONMENT",
"KF_OJ_LOG_LEVEL",
"KF_OJ_BROWSER_ORIGINS",
"KF_OJ_ARTIFACT_ROOT",
"KF_OJ_IMPORT_STAGING_ROOT",
"KF_OJ_ARTIFACT_MAX_BYTES",
"KF_OJ_ARTIFACT_MAX_FILES",
"KF_OJ_DATABASE_URL",
}
ROLE_ENVIRONMENT = {
"api": COMMON_ENVIRONMENT
| {
"KF_OJ_JUDGE_ADMISSION",
"KF_OJ_COORDINATOR_URL",
"KF_OJ_COORDINATOR_TOKEN",
},
"outbox": COMMON_ENVIRONMENT | {"KF_OJ_AMQP_URL"},
"problem-import": COMMON_ENVIRONMENT | {"KF_OJ_IMPORT_WORKER_CGROUP", "KF_OJ_JUDGE_ADMISSION"},
}
def bounded_text(path: Path) -> str:
with path.open() as stream:
contents = stream.read(65537)
if len(contents) > 65536:
raise DeliveryError("process observation exceeds bound")
return contents
def observe_runtime(role: str) -> dict:
group = preflight.current_cgroup()
status = dict(
line.split(":", 1)
for line in bounded_text(Path("/proc/self/status")).splitlines()
if ":" in line
)
limits = {}
for name, identifier in {
"core": resource.RLIMIT_CORE,
"file": resource.RLIMIT_FSIZE,
"files": resource.RLIMIT_NOFILE,
"stack": resource.RLIMIT_STACK,
}.items():
limits[name] = list(resource.getrlimit(identifier))
def read_limits(path):
return {
name: bounded_text(path / name).strip()
for name in ("memory.max", "memory.swap.max", "pids.max", "cpu.max")
}
interfaces = (
[
line.partition(":")[0].strip()
for line in bounded_text(Path("/proc/net/dev")).splitlines()
if ":" in line
]
if role == "problem-import"
else None
)
return {
"cgroup": str(group),
"service_limits": read_limits(group),
"node_limits": read_limits(Path("/sys/fs/cgroup/kfonlinejudge.slice")),
"uids": status.get("Uid", "").split(),
"gids": status.get("Gid", "").split(),
"groups": status.get("Groups", "").split(),
"no_new_privileges": status.get("NoNewPrivs", "").strip(),
"capabilities": [
status.get(key, "").strip()
for key in (
"CapInh",
"CapPrm",
"CapEff",
"CapBnd",
"CapAmb",
)
],
"pid": os.getpid(),
"process_limits": limits,
"interfaces": interfaces,
}
def validate_runtime(role: str, observed: dict) -> None:
if role not in PROFILES:
raise DeliveryError("unknown ordinary service")
user, memory, pids, _, _, _ = PROFILES[role]
uid = str(preflight.ACCOUNTS[user])
if (
observed["cgroup"] != "/sys/fs/cgroup/kfonlinejudge.slice/kf-" + role + ".service"
or observed["uids"] != [uid] * 4
or observed["gids"] != [uid] * 4
or not set(observed["groups"]) <= {uid}
or observed["no_new_privileges"] != "1"
or len(observed["capabilities"]) != 5
or any(not re.fullmatch(r"0+", value) for value in observed["capabilities"])
):
raise DeliveryError("ordinary process identity or privilege boundary differs")
for limits, required_memory, required_pids, cpus in (
(observed["service_limits"], memory, pids, 1),
(observed["node_limits"], 3840, 384, 2),
):
if (
limits["memory.max"] != str(required_memory * 1024**2)
or limits["memory.swap.max"] != "0"
or limits["pids.max"] != str(required_pids)
):
raise DeliveryError("effective ordinary/node memory or process budget differs")
parts = limits["cpu.max"].split()
if len(parts) != 2 or not all(v.isdecimal() for v in parts):
raise DeliveryError("effective CPU budget unavailable")
quota, period = map(int, parts)
if not period or quota != cpus * period:
raise DeliveryError("effective CPU budget differs")
if observed["process_limits"] != {
"core": [0, 0],
"file": [64 * 1024**2] * 2,
"files": [64 if role == "problem-import" else 256] * 2,
"stack": [16 * 1024**2] * 2,
}:
raise DeliveryError("effective ordinary file/descriptor/stack limits differ")
if role in PYTHON_MODULES and observed["pid"] != 1:
raise DeliveryError("private ordinary PID namespace unavailable")
if role == "problem-import" and observed["interfaces"] != ["lo"]:
raise DeliveryError("private import network namespace unavailable")
def python_command(role: str, release: str) -> tuple[list[str], dict]:
payload = Path("/opt/kf-onlinejudge/releases") / release / "payload"
python = payload / "python/bin/python3.13"
preflight.protected(python)
environment = {key: value for key, value in os.environ.items() if key.startswith("KF_OJ_")}
if set(environment) != ROLE_ENVIRONMENT[role] or any(
not value or len(value) > 32768 or any(ch in value for ch in "\r\n\x00")
for value in environment.values()
):
raise DeliveryError("ordinary service environment fields differ")
environment.update(
{
"PATH": "/usr/bin:/bin",
"LC_ALL": "C.UTF-8",
"PYTHONHOME": str(payload / "python"),
"PYTHONPATH": f"{payload}/backend/src:{payload}/site:{payload}/delivery",
"PYTHONDONTWRITEBYTECODE": "1",
"LD_LIBRARY_PATH": f"{payload}/lib:{payload}/python/lib",
}
)
return [str(python), "-s", "-P", "-m", *PYTHON_MODULES[role]], environment
def read_credential(path: Path, expected: str, uid: int) -> bytes:
@@ -72,35 +247,53 @@ def prepare(role: str, release: str, batch: str, directory: Path, expected: str)
os.close(fd)
executable = Path("/opt/kf-onlinejudge/releases") / release / "payload/bin" / binary
preflight.protected(executable)
return [str(executable), *(["serve"] if role == "worker" else []), "--config", str(path)]
return [
str(executable),
*(["serve"] if role == "worker" else []),
"--config",
str(path),
]
def main() -> None:
parser = argparse.ArgumentParser(description="Ordinary service credential bootstrap")
parser.add_argument("role", choices=ROLES)
parser.add_argument("role", choices=PROFILES)
parser.add_argument("--release-id", required=True)
parser.add_argument("--configuration-id", required=True)
parser.add_argument("--credential-directory", required=True, type=Path)
parser.add_argument("--sha256", required=True)
parser.add_argument("--credential-directory", type=Path)
parser.add_argument("--sha256")
args = parser.parse_args()
try:
command = prepare(
args.role,
args.release_id,
args.configuration_id,
args.credential_directory,
args.sha256,
)
if not re.fullmatch(r"sd-[0-9a-f]{12}", args.release_id) or not re.fullmatch(
r"cfg-[0-9a-f]{24}", args.configuration_id
):
raise DeliveryError("invalid ordinary service binding")
validate_runtime(args.role, observe_runtime(args.role))
if args.role in PYTHON_MODULES:
if args.credential_directory is not None or args.sha256 is not None:
raise DeliveryError("unexpected ordinary credential override")
command, environment = python_command(args.role, args.release_id)
else:
if args.credential_directory is None or args.sha256 is None:
raise DeliveryError("fixed ordinary credential required")
command = prepare(
args.role,
args.release_id,
args.configuration_id,
args.credential_directory,
args.sha256,
)
environment = {
"PATH": "/usr/bin:/bin",
"LC_ALL": "C.UTF-8",
"LD_LIBRARY_PATH": str(Path(command[0]).parent.parent / "lib"),
}
# Only the verified normal service binary is launched. No student
# executable or privileged Sandbox operation is accepted here.
os.execve(
command[0],
command,
{
"PATH": "/usr/bin:/bin",
"LC_ALL": "C.UTF-8",
"LD_LIBRARY_PATH": str(Path(command[0]).parent.parent / "lib"),
},
environment,
)
except (OSError, ValueError) as error:
parser.exit(
scripts/release/services.py
+7 -6
View File
@@ -110,13 +110,14 @@ def render(receipt: dict) -> dict[str, bytes]:
"UnsetEnvironment=KF_OJ_DATABASE_URL KF_OJ_IDENTITY_OPERATOR_DATABASE_URL"
" KF_OJ_COORDINATOR_TOKEN KF_OJ_AMQP_URL",
]
elif name == "api":
values += ["ExecStart=" + str(payload / "bin/kf-api")]
elif name == "outbox":
# The original scaffold console module is not a publisher loop.
values += ["ExecStart=" + python + " -s -P -m kf_oj.infra.messaging.publisher"]
else:
values += ["ExecStart=" + str(payload / "bin/kf-problem-import-worker")]
values += [
"ExecStart="
+ python
+ " -s -P -m scripts.release.service_bootstrap "
+ name
+ f" --release-id {release} --configuration-id {batch}"
]
if name in {"api", "problem-import"}:
values += [
"ReadWritePaths=/var/lib/kf-onlinejudge/artifacts /var/lib/kf-onlinejudge/imports"