fix(delivery): wait for HTTPS readiness during startup

scripts/release/README.md

@@ -577,6 +577,14 @@ checks. Each ordinary helper has one CPU within the node's two-CPU ceiling;
 its memory, process and privilege limits remain fixed. These operator budgets
 do not change student execution budgets. Startup opens admission only after all
 required readiness queries succeed.
+The proxy's `Type=exec` state only confirms its launcher is running. Lifecycle
+startup and resume confirm the real pinned HTTPS/API response within the existing
+130-second service-confirmation deadline. Only connection refusal, timeouts and
+upstream 502/503/504 observations retry within that same deadline. Certificate,
+hostname, configuration and response-validation failures still refuse startup;
+ordinary health queries remain single observations. No service, node or student
+resource limit is increased.
+
 
 Database authority queries use the fixed local TCP/Unix endpoints with TLS disabled
 and a five-second connection deadline. SQL remains read-only with a two-second

scripts/release/lifecycle.py

@@ -22,6 +22,7 @@ from . import (
     operation_proof,
     operator_units,
     preflight,
+    proxy_health,
     proxy_install,
     proxy_service,
     service_install,
@@ -372,6 +373,8 @@ class Maintenance:
         values = self.unit(name)
         if operation == "start" and values["ActiveState"] == "active":
             node_health.inspect_unit(name, self.selected)
+            if name == "proxy":
+                self._wait_for_proxy(time.monotonic() + 130)
             return
         if operation == "stop" and values["ActiveState"] == "inactive" and values["MainPID"] == "0":
             self.empty_cgroup(name)
@@ -396,6 +399,8 @@ class Maintenance:
                 and values["SubState"] == "running"
             ):
                 node_health.inspect_unit(name, self.selected)
+                if name == "proxy":
+                    self._wait_for_proxy(deadline)
                 break
             if (
                 operation == "stop"
@@ -410,6 +415,40 @@ class Maintenance:
         self.record["events"][-1]["confirmed"] = True
         self.save(operation + "_confirmed")
 
+    def _wait_for_proxy(self, deadline: float) -> None:
+        """Type=exec is process startup; confirm actual HTTPS before admission."""
+        while True:
+            if time.monotonic() >= deadline:
+                raise DeliveryError("bounded HTTPS startup unconfirmed")
+            if install.selection(check_configuration=False) != self.selected:
+                raise DeliveryError("selected binding changed during maintenance")
+            # Invalid selected TLS/configuration is fatal, never a retry reason.
+            proxy_install.check_binding(
+                self.selected["release_id"], self.selected["configuration_id"]
+            )
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                raise DeliveryError("bounded HTTPS startup unconfirmed")
+            try:
+                ready = proxy_health.observe(self.selected, timeout=min(5.0, remaining))
+            except proxy_health.HTTPSNotReady as error:
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    raise DeliveryError("bounded HTTPS startup unconfirmed") from error
+                time.sleep(min(0.5, remaining))
+                continue
+            if time.monotonic() >= deadline:
+                raise DeliveryError("bounded HTTPS startup unconfirmed")
+            if install.selection(check_configuration=False) != self.selected:
+                raise DeliveryError("selected binding changed during maintenance")
+            if ready != {
+                "reachable": True,
+                "pinned_certificate_and_hostname_verified": True,
+                "public_browser_trust_verified": False,
+            }:
+                raise DeliveryError("HTTPS startup proof differs")
+            return
+
     def start_group(self, names: tuple[str, ...]) -> None:
         """Dispatch one fixed dependency stage; confirm each real unit separately."""
         if names not in START_GROUPS:
@@ -424,6 +463,8 @@ class Maintenance:
             values = self.unit(name)
             if values["ActiveState"] == "active":
                 node_health.inspect_unit(name, self.selected)
+                if name == "proxy":
+                    self._wait_for_proxy(time.monotonic() + 130)
             else:
                 inactive.append(name)
         pending = {}
@@ -448,6 +489,8 @@ class Maintenance:
                     raise DeliveryError("bounded lifecycle operation unconfirmed")
                 if values["ActiveState"] == "active" and values["SubState"] == "running":
                     node_health.inspect_unit(name, self.selected)
+                    if name == "proxy":
+                        self._wait_for_proxy(deadline)
                     event["confirmed"] = True
                     self.save("start_confirmed")
                     del pending[name]

scripts/release/proxy_health.py

@@ -10,7 +10,20 @@ from . import proxy_service
 from .policy import DeliveryError, read_json
 
 
-def observe(selected: dict) -> dict:
+class HTTPSNotReady(DeliveryError):
+    """A bounded transport or upstream startup observation is unavailable."""
+
+
+def observe(selected: dict, *, timeout: float = 5.0) -> dict:
+    if type(timeout) not in (int, float) or not 0 < timeout <= 5:
+        raise DeliveryError("HTTPS observation timeout exceeds bound")
+    try:
+        return _observe(selected, timeout=timeout)
+    except (ConnectionRefusedError, TimeoutError) as error:
+        raise HTTPSNotReady("HTTPS listener or API is not ready") from error
+
+
+def _observe(selected: dict, *, timeout: float) -> dict:
     directory = proxy_service.batch(selected)
     spec = read_json(directory / "launch.json", 4096)
     host = urlsplit(spec["origin"]).hostname
@@ -22,9 +35,9 @@ def observe(selected: dict) -> dict:
     ctx.load_verify_locations(cafile=certificate)
     ctx.verify_flags |= ssl.VERIFY_X509_PARTIAL_CHAIN
     ctx.minimum_version = ssl.TLSVersion.TLSv1_2
-    with socket.create_connection(("127.0.0.1", 443), timeout=5) as raw:
+    with socket.create_connection(("127.0.0.1", 443), timeout=timeout) as raw:
         with ctx.wrap_socket(raw, server_hostname=host) as client:
-            client.settimeout(5)
+            client.settimeout(timeout)
             der = client.getpeercert(binary_form=True)
             pem = certificate.read_text()
             leaf = pem[: pem.index("-----END CERTIFICATE-----") + len("-----END CERTIFICATE-----")]
@@ -41,6 +54,8 @@ def observe(selected: dict) -> dict:
             response = http.client.HTTPResponse(client)
             try:
                 response.begin()
+                if response.status in (502, 503, 504):
+                    raise HTTPSNotReady("HTTPS API upstream is not ready")
                 body = response.read(4097)
                 if response.status != 200 or len(body) > 4096:
                     raise DeliveryError("HTTPS health response exceeds bound or is unavailable")