fix(delivery): wait for HTTPS readiness during startup

.github/workflows/engineering.yml

@@ -109,6 +109,7 @@ jobs:
             tests/test_production_logging.py \
             tests/test_delivery.py \
             tests/test_delivery_io.py \
+            tests/test_delivery_proxy_startup.py \
             tests/test_delivery_validation_work.py \
             tests/test_delivery_bytecode.py \
             tests/test_delivery_backup.py \

backend/tests/test_delivery_io.py

@@ -1123,6 +1123,24 @@ def ordinary_lifecycle_model(monkeypatch):
     monkeypatch.setattr(
         lifecycle.node_health, "inspect_unit", lambda name, _: trace.append(("inspect", name))
     )
+    # This fixture remains an ordinary model; actual TLS/service readiness is VM-only.
+    monkeypatch.setattr(
+        lifecycle.proxy_install,
+        "check_binding",
+        lambda *_: trace.append(("proxy-binding", "proxy")),
+    )
+
+    def https_ready(selected, *, timeout):
+        assert selected == maintenance.selected and 0 < timeout <= 5
+        assert states["proxy"] == "active"
+        trace.append(("https", "proxy"))
+        return {
+            "reachable": True,
+            "pinned_certificate_and_hostname_verified": True,
+            "public_browser_trust_verified": False,
+        }
+
+    monkeypatch.setattr(lifecycle.proxy_health, "observe", https_ready)
     monkeypatch.setattr(lifecycle.time, "monotonic", lambda: clock[0])
     monkeypatch.setattr(
         lifecycle.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds)
@@ -1327,12 +1345,14 @@ def test_startup_dependency_barriers_precede_both_real_readiness_calls(
     def observe(*args):
         verified = {name for kind, name in trace if kind == "inspect"}
         assert verified == set(states) and all(value == "active" for value in states.values())
+        trace.append(("health", gate[-1]))
         observations.append(gate[-1])
         return {"service_preconditions_met": True, "ready_for_admission": gate[-1] == "open"}
 
     monkeypatch.setattr(lifecycle.operator_units, "run", observe)
     maintenance.start_locked(resume=True)
     assert gate == observations == ["draining", "open"]
+    assert trace.index(("https", "proxy")) < trace.index(("health", "draining"))
     dispatches = [name for kind, name in trace if kind == "dispatch"]
     assert dispatches == [*lifecycle.DEPENDENCIES, *lifecycle.START_ORDER]
     for dependency in (*lifecycle.DEPENDENCIES, "sandbox-factory"):

backend/tests/test_delivery_proxy_startup.py

@@ -0,0 +1,140 @@
+"""Ordinary modeled lifecycle checks; actual HTTPS and service gates require the VM."""
+
+import importlib
+import ssl
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+lifecycle = importlib.import_module("scripts.release.lifecycle")
+proxy_health = importlib.import_module("scripts.release.proxy_health")
+DeliveryError = importlib.import_module("scripts.release.policy").DeliveryError
+READY = {
+    "reachable": True,
+    "pinned_certificate_and_hostname_verified": True,
+    "public_browser_trust_verified": False,
+}
+
+
+def setup_maintenance(monkeypatch, *, active=False):
+    maintenance = object.__new__(lifecycle.Maintenance)
+    maintenance.selected = {"release_id": "sd-123456789abc", "configuration_id": "cfg-" + "1" * 24}
+    maintenance.record = {"events": []}
+    clock = [100.0]
+    state = {"active": active}
+    saves = []
+    monkeypatch.setattr(lifecycle.time, "monotonic", lambda: clock[0])
+    monkeypatch.setattr(
+        lifecycle.time, "sleep", lambda value: clock.__setitem__(0, clock[0] + value)
+    )
+    monkeypatch.setattr(lifecycle.install, "selection", lambda **_: maintenance.selected.copy())
+    monkeypatch.setattr(lifecycle.node_health, "inspect_unit", lambda *_: None)
+    monkeypatch.setattr(lifecycle.proxy_install, "check_binding", lambda *_: None)
+    monkeypatch.setattr(
+        maintenance,
+        "unit",
+        lambda _: {
+            "ActiveState": "active" if state["active"] else "inactive",
+            "SubState": "running",
+            "MainPID": "42" if state["active"] else "0",
+        },
+    )
+    monkeypatch.setattr(
+        maintenance,
+        "save",
+        lambda phase: saves.append((phase, [x.copy() for x in maintenance.record["events"]])),
+    )
+
+    def systemctl(*args):
+        assert args == ("--no-block", "start", "kf-proxy.service")
+        state["active"] = True
+        return ""
+
+    monkeypatch.setattr(lifecycle.service_install, "systemctl", systemctl)
+    return maintenance, clock, saves
+
+
+@pytest.mark.parametrize(
+    "group,active", [(True, False), (False, False), (True, True), (False, True)]
+)
+def test_proxy_process_is_not_confirmed_until_actual_https_probe_succeeds(
+    monkeypatch, group, active
+):
+    maintenance, clock, saves = setup_maintenance(monkeypatch, active=active)
+    probes = []
+
+    def probe(selected, *, timeout):
+        assert all(not x["confirmed"] for x in maintenance.record["events"])
+        probes.append(timeout)
+        if len(probes) == 1:
+            raise proxy_health.HTTPSNotReady("launcher has not served HTTPS yet")
+        return READY.copy()
+
+    monkeypatch.setattr(proxy_health, "observe", probe)
+    if group:
+        maintenance.start_group(("proxy",))
+    else:
+        maintenance.change("proxy", "start")
+    assert probes == [5.0, 5.0] and clock[0] == 100.5
+    if not active:
+        assert saves[0][1][0]["confirmed"] is False
+        assert saves[-1][1][0]["confirmed"] is True
+
+
+@pytest.mark.parametrize(
+    "error", [ssl.SSLCertVerificationError("bad certificate"), DeliveryError("changed TLS binding")]
+)
+def test_proxy_security_failure_is_fatal_without_wait_or_confirmation(monkeypatch, error):
+    maintenance, clock, _ = setup_maintenance(monkeypatch)
+    calls = []
+
+    def probe(*_, **__):
+        calls.append(1)
+        raise error
+
+    monkeypatch.setattr(proxy_health, "observe", probe)
+    with pytest.raises(type(error)):
+        maintenance.start_group(("proxy",))
+    assert calls == [1] and clock[0] == 100
+    assert maintenance.record["events"] == [
+        {"unit": "proxy", "operation": "start", "confirmed": False}
+    ]
+
+
+@pytest.mark.parametrize("late_ready", [False, True])
+def test_proxy_wait_cannot_restart_or_succeed_after_original_deadline(monkeypatch, late_ready):
+    maintenance, clock, _ = setup_maintenance(monkeypatch)
+    probes = []
+
+    def unavailable(*_, timeout):
+        probes.append(timeout)
+        clock[0] += 131
+        if late_ready:
+            return READY.copy()
+        raise proxy_health.HTTPSNotReady("still unavailable")
+
+    monkeypatch.setattr(proxy_health, "observe", unavailable)
+    with pytest.raises(DeliveryError, match="bounded HTTPS startup unconfirmed"):
+        maintenance.start_group(("proxy",))
+    assert probes == [5.0]
+    assert maintenance.record["events"][0]["confirmed"] is False
+
+
+@pytest.mark.parametrize(
+    "error", [ConnectionRefusedError("not listening"), TimeoutError("not ready")]
+)
+def test_proxy_probe_labels_only_transient_transport_failure(monkeypatch, error):
+    def unavailable(*_, **__):
+        raise error
+
+    monkeypatch.setattr(proxy_health, "_observe", unavailable)
+    with pytest.raises(proxy_health.HTTPSNotReady):
+        proxy_health.observe({})
+
+
+@pytest.mark.parametrize("timeout", [0, -1, 5.01, True, float("nan")])
+def test_proxy_probe_rejects_unbounded_timeout(timeout):
+    with pytest.raises(DeliveryError, match="timeout exceeds bound"):
+        proxy_health.observe({}, timeout=timeout)

scripts/release/README.md

@@ -577,6 +577,14 @@ checks. Each ordinary helper has one CPU within the node's two-CPU ceiling;
 its memory, process and privilege limits remain fixed. These operator budgets
 do not change student execution budgets. Startup opens admission only after all
 required readiness queries succeed.
+The proxy's `Type=exec` state only confirms its launcher is running. Lifecycle
+startup and resume confirm the real pinned HTTPS/API response within the existing
+130-second service-confirmation deadline. Only connection refusal, timeouts and
+upstream 502/503/504 observations retry within that same deadline. Certificate,
+hostname, configuration and response-validation failures still refuse startup;
+ordinary health queries remain single observations. No service, node or student
+resource limit is increased.
+
 
 Database authority queries use the fixed local TCP/Unix endpoints with TLS disabled
 and a five-second connection deadline. SQL remains read-only with a two-second

scripts/release/lifecycle.py

@@ -22,6 +22,7 @@ from . import (
     operation_proof,
     operator_units,
     preflight,
+    proxy_health,
     proxy_install,
     proxy_service,
     service_install,
@@ -372,6 +373,8 @@ class Maintenance:
         values = self.unit(name)
         if operation == "start" and values["ActiveState"] == "active":
             node_health.inspect_unit(name, self.selected)
+            if name == "proxy":
+                self._wait_for_proxy(time.monotonic() + 130)
             return
         if operation == "stop" and values["ActiveState"] == "inactive" and values["MainPID"] == "0":
             self.empty_cgroup(name)
@@ -396,6 +399,8 @@ class Maintenance:
                 and values["SubState"] == "running"
             ):
                 node_health.inspect_unit(name, self.selected)
+                if name == "proxy":
+                    self._wait_for_proxy(deadline)
                 break
             if (
                 operation == "stop"
@@ -410,6 +415,40 @@ class Maintenance:
         self.record["events"][-1]["confirmed"] = True
         self.save(operation + "_confirmed")
 
+    def _wait_for_proxy(self, deadline: float) -> None:
+        """Type=exec is process startup; confirm actual HTTPS before admission."""
+        while True:
+            if time.monotonic() >= deadline:
+                raise DeliveryError("bounded HTTPS startup unconfirmed")
+            if install.selection(check_configuration=False) != self.selected:
+                raise DeliveryError("selected binding changed during maintenance")
+            # Invalid selected TLS/configuration is fatal, never a retry reason.
+            proxy_install.check_binding(
+                self.selected["release_id"], self.selected["configuration_id"]
+            )
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                raise DeliveryError("bounded HTTPS startup unconfirmed")
+            try:
+                ready = proxy_health.observe(self.selected, timeout=min(5.0, remaining))
+            except proxy_health.HTTPSNotReady as error:
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    raise DeliveryError("bounded HTTPS startup unconfirmed") from error
+                time.sleep(min(0.5, remaining))
+                continue
+            if time.monotonic() >= deadline:
+                raise DeliveryError("bounded HTTPS startup unconfirmed")
+            if install.selection(check_configuration=False) != self.selected:
+                raise DeliveryError("selected binding changed during maintenance")
+            if ready != {
+                "reachable": True,
+                "pinned_certificate_and_hostname_verified": True,
+                "public_browser_trust_verified": False,
+            }:
+                raise DeliveryError("HTTPS startup proof differs")
+            return
+
     def start_group(self, names: tuple[str, ...]) -> None:
         """Dispatch one fixed dependency stage; confirm each real unit separately."""
         if names not in START_GROUPS:
@@ -424,6 +463,8 @@ class Maintenance:
             values = self.unit(name)
             if values["ActiveState"] == "active":
                 node_health.inspect_unit(name, self.selected)
+                if name == "proxy":
+                    self._wait_for_proxy(time.monotonic() + 130)
             else:
                 inactive.append(name)
         pending = {}
@@ -448,6 +489,8 @@ class Maintenance:
                     raise DeliveryError("bounded lifecycle operation unconfirmed")
                 if values["ActiveState"] == "active" and values["SubState"] == "running":
                     node_health.inspect_unit(name, self.selected)
+                    if name == "proxy":
+                        self._wait_for_proxy(deadline)
                     event["confirmed"] = True
                     self.save("start_confirmed")
                     del pending[name]

scripts/release/proxy_health.py

@@ -10,7 +10,20 @@ from . import proxy_service
 from .policy import DeliveryError, read_json
 
 
-def observe(selected: dict) -> dict:
+class HTTPSNotReady(DeliveryError):
+    """A bounded transport or upstream startup observation is unavailable."""
+
+
+def observe(selected: dict, *, timeout: float = 5.0) -> dict:
+    if type(timeout) not in (int, float) or not 0 < timeout <= 5:
+        raise DeliveryError("HTTPS observation timeout exceeds bound")
+    try:
+        return _observe(selected, timeout=timeout)
+    except (ConnectionRefusedError, TimeoutError) as error:
+        raise HTTPSNotReady("HTTPS listener or API is not ready") from error
+
+
+def _observe(selected: dict, *, timeout: float) -> dict:
     directory = proxy_service.batch(selected)
     spec = read_json(directory / "launch.json", 4096)
     host = urlsplit(spec["origin"]).hostname
@@ -22,9 +35,9 @@ def observe(selected: dict) -> dict:
     ctx.load_verify_locations(cafile=certificate)
     ctx.verify_flags |= ssl.VERIFY_X509_PARTIAL_CHAIN
     ctx.minimum_version = ssl.TLSVersion.TLSv1_2
-    with socket.create_connection(("127.0.0.1", 443), timeout=5) as raw:
+    with socket.create_connection(("127.0.0.1", 443), timeout=timeout) as raw:
         with ctx.wrap_socket(raw, server_hostname=host) as client:
-            client.settimeout(5)
+            client.settimeout(timeout)
             der = client.getpeercert(binary_form=True)
             pem = certificate.read_text()
             leaf = pem[: pem.index("-----END CERTIFICATE-----") + len("-----END CERTIFICATE-----")]
@@ -41,6 +54,8 @@ def observe(selected: dict) -> dict:
             response = http.client.HTTPResponse(client)
             try:
                 response.begin()
+                if response.status in (502, 503, 504):
+                    raise HTTPSNotReady("HTTPS API upstream is not ready")
                 body = response.read(4097)
                 if response.status != 200 or len(body) > 4096:
                     raise DeliveryError("HTTPS health response exceeds bound or is unavailable")