"""Finite fixed-purpose operator helpers; no caller-controlled executable or identity.""" import hashlib import json import os import re import resource import subprocess from pathlib import Path from . import preflight from .policy import DeliveryError, invalid_constant, unique_object class OperatorExecutionError(DeliveryError): """A fixed helper was dispatched but its result or cleanup was unconfirmed.""" def __init__(self, message: str, failures: list | None = None): super().__init__(message) self.failures = failures or [] PROFILES = { "node-health": ("root", 0, 128, 16, 50, 300), "broker-health": ("rabbitmq", 1204, 256, 64, 100, 45), "factory-health": ("kf-worker", 1202, 32, 4, 100, 30), "import-health": ("kf-oj", 1200, 128, 16, 100, 60), "outbox-health": ("kf-oj", 1200, 128, 16, 100, 60), "node-maintenance": ("root", 0, 256, 32, 50, 900), "node-observation": ("root", 0, 128, 16, 25, 360), } # OTP 27 reserves a 32 MiB dual-mapped JIT block before serving CLI queries. FILE_LIMIT_MIB = {"broker-health": 64} DIAGNOSTIC_PREFIX = "KF_OPERATOR_DIAGNOSTIC " READINESS_ERRORS = frozenset( { "bounded_journal_unavailable", "https_entry_unavailable", "selected_binding_changed", "db_authority_unavailable", "api_live_unavailable", "execution_owner_unavailable", "broker_metadata_unavailable", "outbox_pipeline_unavailable", "import_pipeline_unavailable", "factory_status_unavailable", "notification_delivery_unconfirmed", *( name + "_unavailable" for name in ( "api", "coordinator", "outbox", "problem-import", "worker", "postgres", "rabbitmq", "sandbox-factory", "proxy", ) ), } ) def failure_record(name: str, outcome: str, result=None, error=None) -> dict: """Record only fixed identifiers, sizes and digests; never child output or argv.""" record = {"helper": name, "outcome": outcome, "returncode": None} if result is not None: record["returncode"] = result.returncode for stream in ("stdout", "stderr"): value = getattr(result, stream, b"") if result is not None else b"" if isinstance(error, subprocess.TimeoutExpired): value = getattr(error, stream, None) or b"" record[stream + "_bytes"] = len(value) record[stream + "_sha256"] = hashlib.sha256(value).hexdigest() record["error_sha256"] = hashlib.sha256( (str(error) if isinstance(error, DeliveryError) else type(error).__name__).encode() ).hexdigest() return record def diagnostic_chain(stderr: bytes) -> list: """Accept one small, strictly shaped record from a fixed child CLI.""" if len(stderr) > 4096: return [] lines = [line for line in stderr.splitlines() if line.startswith(DIAGNOSTIC_PREFIX.encode())] try: if len(lines) != 1: return [] value = decode(lines[0][len(DIAGNOSTIC_PREFIX) :], 3072) if ( set(value) != {"schema_version", "failures"} or type(value["schema_version"]) is not int or value["schema_version"] != 1 ): return [] failures = value["failures"] if not isinstance(failures, list) or not 1 <= len(failures) <= 3: return [] fields = { "helper", "outcome", "returncode", "stdout_bytes", "stdout_sha256", "stderr_bytes", "stderr_sha256", "error_sha256", } for row in failures: if not isinstance(row, dict) or set(row) not in ( fields, fields | {"readiness_errors"}, ): return [] if row["helper"] not in PROFILES or row["outcome"] not in { "dispatch", "exit", "decode", "cleanup", "timeout", "preconditions", "admission", }: return [] code = row["returncode"] if code is not None and (type(code) is not int or not -255 <= code <= 255): return [] if any( type(row[k]) is not int or not 0 <= row[k] <= 2**63 - 1 for k in ("stdout_bytes", "stderr_bytes") ): return [] if any( not isinstance(row[k], str) or not re.fullmatch("[a-f0-9]{64}", row[k]) for k in ("stdout_sha256", "stderr_sha256", "error_sha256") ): return [] errors = row.get("readiness_errors", []) if ( not isinstance(errors, list) or len(errors) > len(READINESS_ERRORS) or any(not isinstance(code, str) or code not in READINESS_ERRORS for code in errors) ): return [] return failures except (TypeError, ValueError, KeyError): return [] def diagnostic_suffix(error: Exception) -> str: if not isinstance(error, OperatorExecutionError) or not error.failures: return "" # Revalidate even internally built records before they cross a CLI boundary. value = json.dumps({"schema_version": 1, "failures": error.failures}, separators=(",", ":")) line = DIAGNOSTIC_PREFIX + value if len(error.failures) > 4 or len(line) > 3072: return "" # The child decoder allows three ancestors; validate the fourth separately. for row in error.failures: single = DIAGNOSTIC_PREFIX + json.dumps({"schema_version": 1, "failures": [row]}) if not diagnostic_chain(single.encode()): return "" return line + "\n" def readiness_failure(message: str, result: dict, *, admission: bool = False): error = OperatorExecutionError(message) row = failure_record("node-health", "admission" if admission else "preconditions", error=error) errors = result.get("errors") if ( isinstance(errors, list) and len(errors) <= len(READINESS_ERRORS) and all(isinstance(code, str) and code in READINESS_ERRORS for code in errors) ): row["readiness_errors"] = errors child = DIAGNOSTIC_PREFIX + json.dumps( {"schema_version": 1, "failures": result.get("operator_failures", [])} ) error.failures = diagnostic_chain(child.encode())[:2] + [row] return error def guarded(name: str) -> None: _, uid, memory, pids, cpu, _ = PROFILES[name] if ( os.getuid() != uid or os.geteuid() != uid or os.getgid() != uid or not set(os.getgroups()) <= {uid} ): raise DeliveryError("fixed operator helper identity required") observed = dict( line.split(":", 1) for line in Path("/proc/self/status").read_text().splitlines() if ":" in line ) if observed.get("NoNewPrivs", "").strip() != "1" or any( not observed.get(k, "").strip() or int(observed[k].strip(), 16) != 0 for k in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb") ): raise DeliveryError("operator helper privilege boundary differs") group = preflight.current_cgroup() if str(group) != "/sys/fs/cgroup/kfonlinejudge.slice/kf-" + name + ".service": raise DeliveryError("operator helper cgroup differs") for directory, expected in ( (group, [str(memory * 1024**2), "0", str(pids), f"{cpu * 1000} 100000"]), ( Path("/sys/fs/cgroup/kfonlinejudge.slice"), [str(3840 * 1024**2), "0", "384", "200000 100000"], ), ): actual = [ (directory / k).read_text().strip() for k in ("memory.max", "memory.swap.max", "pids.max", "cpu.max") ] if actual != expected: raise DeliveryError("operator helper/node budgets differ") for key, maximum in ( (resource.RLIMIT_CORE, 0), (resource.RLIMIT_FSIZE, FILE_LIMIT_MIB.get(name, 16) * 1024**2), (resource.RLIMIT_NOFILE, 256), (resource.RLIMIT_STACK, 16 * 1024**2), ): if resource.getrlimit(key) != (maximum, maximum): raise DeliveryError("operator helper process budgets differ") def decode(contents: bytes, maximum: int = 32768): if len(contents) > maximum: raise DeliveryError("operator output exceeds bound") return json.loads( contents, object_pairs_hook=unique_object, parse_constant=invalid_constant, ) def run( name: str, payload: Path, *, notification_probe: bool = False, action: str | None = None, ) -> dict: from . import configuration, install, service_install from .services import python_environment if os.getuid() != 0 or os.geteuid() != 0 or name not in PROFILES: raise DeliveryError("fixed root operator helper required") if name == "node-maintenance": if action not in {"start", "resume", "drain", "stop", "restart"} or notification_probe: raise DeliveryError("explicit fixed maintenance action required") elif action is not None: raise DeliveryError("unexpected operator action override") user, _, memory, pids, cpu, seconds = PROFILES[name] unit = "kf-" + name + ".service" current = dict( line.split("=", 1) for line in service_install.systemctl( "show", unit, "--property=LoadState,ActiveState,MainPID" ).splitlines() if "=" in line ) if ( current.get("LoadState") != "not-found" or current.get("ActiveState") != "inactive" or current.get("MainPID") != "0" ): raise DeliveryError("operator helper already active or unknown") release_id = payload.parent.name selected = install.selection(check_configuration=False) if ( payload != install.RELEASES / selected["release_id"] / "payload" or release_id != selected["release_id"] ): raise DeliveryError("operator helper is not the selected payload") description = "KF fixed " + name + " " + release_id + "." + selected["configuration_id"] command = [ "/usr/bin/systemd-run", "--quiet", "--wait", "--pipe", "--collect", "--unit=" + unit, "--description=" + description, "--slice=kfonlinejudge.slice", ] properties = { "Type": "exec", "User": user, "Group": user, "UMask": "0077", "MemoryMax": str(memory) + "M", "MemorySwapMax": "0", "TasksMax": str(pids), "CPUQuota": str(cpu) + "%", "LimitCORE": "0", "LimitFSIZE": str(FILE_LIMIT_MIB.get(name, 16)) + "M", "LimitNOFILE": "256", "LimitSTACK": "16M", "RuntimeMaxSec": str(seconds), "TimeoutStartSec": "10", "TimeoutStopSec": "10", "Restart": "no", "KillMode": "control-group", "NoNewPrivileges": "yes", "CapabilityBoundingSet": "", "AmbientCapabilities": "", "ProtectSystem": "strict", "ProtectHome": "yes", "PrivateTmp": "yes", "PrivateDevices": "yes", "ProtectControlGroups": "yes", "RestrictAddressFamilies": "AF_UNIX AF_INET", "WorkingDirectory": str(payload), } if name == "node-observation": # The unified io controller may be disabled until a unit requests # accounting. Enable it through systemd before reading io.stat for # this slice and its services; keep missing counters a hard failure. properties["IOAccounting"] = "yes" for key, value in properties.items(): command.append("--property=" + key + "=" + value) if name in {"import-health", "outbox-health"}: path = configuration.installed_path( selected["release_id"], selected["configuration_id"], "import.env" if name == "import-health" else "outbox.env", ) command += [ "--property=EnvironmentFile=" + str(path), "--property=PrivatePIDs=yes", ] if name == "import-health": command += [ "--property=PrivateNetwork=yes", "--property=RestrictAddressFamilies=AF_UNIX", ] if name == "node-maintenance": command += [ "--property=RuntimeDirectory=kf-node-verification", "--property=RuntimeDirectoryMode=0700", "--property=ReadWritePaths=/etc/kf-admission /var/lib/kf-onlinejudge/operations" " /run/kf-node-verification", ] if name in {"import-health", "outbox-health"}: if notification_probe: raise DeliveryError("unexpected import probe override") command += [ "--property=Environment=PATH=/usr/bin:/bin LC_ALL=C.UTF-8 " + python_environment(payload) ] command += [ str(payload / "python/bin/python3.13"), "-s", "-P", "-m", "scripts.release." + name.replace("-", "_"), "--inside", ] else: command += ["/usr/bin/env", "-i", "PATH=/usr/bin:/bin", "LC_ALL=C.UTF-8"] if name == "factory-health": if notification_probe: raise DeliveryError("unexpected factory probe override") command += [ *python_environment(payload).split(), str(payload / "python/bin/python3.13"), "-s", "-P", "-m", "scripts.release.factory_health", "--inside", ] elif name not in {"import-health", "outbox-health"}: command += [ *python_environment(payload).split(), str(payload / "python/bin/python3.13"), "-s", "-P", "-m", "scripts.release.lifecycle" if name == "node-maintenance" else "scripts.release." + name.replace("-", "_"), "--inside", ] if name == "node-maintenance": command += ["--action", action] if notification_probe and name == "node-health": command.append("--probe-notification") elif notification_probe: raise DeliveryError("unexpected broker probe override") result = None outcome = "dispatch" try: try: # Pipe output is bounded by the fixed helper and the service file/memory # ceilings; helper tools below cap each pipe before serializing it. result = subprocess.run( command, capture_output=True, timeout=seconds + 15, env={"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, ) outcome = "exit" if result.returncode or len(result.stderr) > 4096: raise DeliveryError("operator helper unavailable") outcome = "decode" value = decode(result.stdout) if not isinstance(value, dict): raise DeliveryError("operator helper object required") finally: previous_outcome, outcome = outcome, "cleanup" actual = dict( line.split("=", 1) for line in service_install.systemctl( "show", unit, "--property=LoadState,ActiveState,MainPID,Transient,Description", ).splitlines() if "=" in line ) if actual.get("LoadState") != "not-found": if actual.get("Transient") != "yes" or actual.get("Description") != description: raise DeliveryError("foreign operator helper; refused cleanup") service_install.systemctl("stop", unit) actual = dict( line.split("=", 1) for line in service_install.systemctl( "show", unit, "--property=ActiveState,MainPID" ).splitlines() if "=" in line ) if actual.get("ActiveState") != "inactive" or actual.get("MainPID") != "0": raise DeliveryError("operator helper cleanup unconfirmed") outcome = previous_outcome except (OSError, ValueError, subprocess.SubprocessError) as error: message = str(error) if isinstance(error, DeliveryError) else type(error).__name__ if isinstance(error, subprocess.TimeoutExpired) and outcome != "cleanup": outcome = "timeout" failures = diagnostic_chain(result.stderr) if result is not None else [] failures.append(failure_record(name, outcome, result, error)) raise OperatorExecutionError(message, failures) from error return value