From 6e20b530134fc0655e7185cdfedfdfe73654e11d Mon Sep 17 00:00:00 2001 From: kronecker <2026692687@qq.com> Date: Thu, 29 Oct 2026 15:29:50 +0800 Subject: [PATCH] fix(delivery): wait for HTTPS readiness during startup --- .github/workflows/engineering.yml | 1 + backend/tests/test_delivery_io.py | 20 +++ backend/tests/test_delivery_proxy_startup.py | 140 +++++++++++++++++++ scripts/release/README.md | 8 ++ scripts/release/lifecycle.py | 43 ++++++ scripts/release/proxy_health.py | 21 ++- 6 files changed, 230 insertions(+), 3 deletions(-) create mode 100644 backend/tests/test_delivery_proxy_startup.py diff --git a/.github/workflows/engineering.yml b/.github/workflows/engineering.yml index 55e4839..ccc3de5 100644 --- a/.github/workflows/engineering.yml +++ b/.github/workflows/engineering.yml @@ -109,6 +109,7 @@ jobs: tests/test_production_logging.py \ tests/test_delivery.py \ tests/test_delivery_io.py \ + tests/test_delivery_proxy_startup.py \ tests/test_delivery_validation_work.py \ tests/test_delivery_bytecode.py \ tests/test_delivery_backup.py \ diff --git a/backend/tests/test_delivery_io.py b/backend/tests/test_delivery_io.py index 426d396..8c9a491 100644 --- a/backend/tests/test_delivery_io.py +++ b/backend/tests/test_delivery_io.py @@ -1123,6 +1123,24 @@ def ordinary_lifecycle_model(monkeypatch): monkeypatch.setattr( lifecycle.node_health, "inspect_unit", lambda name, _: trace.append(("inspect", name)) ) + # This fixture remains an ordinary model; actual TLS/service readiness is VM-only. + monkeypatch.setattr( + lifecycle.proxy_install, + "check_binding", + lambda *_: trace.append(("proxy-binding", "proxy")), + ) + + def https_ready(selected, *, timeout): + assert selected == maintenance.selected and 0 < timeout <= 5 + assert states["proxy"] == "active" + trace.append(("https", "proxy")) + return { + "reachable": True, + "pinned_certificate_and_hostname_verified": True, + "public_browser_trust_verified": False, + } + + monkeypatch.setattr(lifecycle.proxy_health, "observe", https_ready) monkeypatch.setattr(lifecycle.time, "monotonic", lambda: clock[0]) monkeypatch.setattr( lifecycle.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds) @@ -1327,12 +1345,14 @@ def test_startup_dependency_barriers_precede_both_real_readiness_calls( def observe(*args): verified = {name for kind, name in trace if kind == "inspect"} assert verified == set(states) and all(value == "active" for value in states.values()) + trace.append(("health", gate[-1])) observations.append(gate[-1]) return {"service_preconditions_met": True, "ready_for_admission": gate[-1] == "open"} monkeypatch.setattr(lifecycle.operator_units, "run", observe) maintenance.start_locked(resume=True) assert gate == observations == ["draining", "open"] + assert trace.index(("https", "proxy")) < trace.index(("health", "draining")) dispatches = [name for kind, name in trace if kind == "dispatch"] assert dispatches == [*lifecycle.DEPENDENCIES, *lifecycle.START_ORDER] for dependency in (*lifecycle.DEPENDENCIES, "sandbox-factory"): diff --git a/backend/tests/test_delivery_proxy_startup.py b/backend/tests/test_delivery_proxy_startup.py new file mode 100644 index 0000000..0af4fe9 --- /dev/null +++ b/backend/tests/test_delivery_proxy_startup.py @@ -0,0 +1,140 @@ +"""Ordinary modeled lifecycle checks; actual HTTPS and service gates require the VM.""" + +import importlib +import ssl +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +lifecycle = importlib.import_module("scripts.release.lifecycle") +proxy_health = importlib.import_module("scripts.release.proxy_health") +DeliveryError = importlib.import_module("scripts.release.policy").DeliveryError +READY = { + "reachable": True, + "pinned_certificate_and_hostname_verified": True, + "public_browser_trust_verified": False, +} + + +def setup_maintenance(monkeypatch, *, active=False): + maintenance = object.__new__(lifecycle.Maintenance) + maintenance.selected = {"release_id": "sd-123456789abc", "configuration_id": "cfg-" + "1" * 24} + maintenance.record = {"events": []} + clock = [100.0] + state = {"active": active} + saves = [] + monkeypatch.setattr(lifecycle.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr( + lifecycle.time, "sleep", lambda value: clock.__setitem__(0, clock[0] + value) + ) + monkeypatch.setattr(lifecycle.install, "selection", lambda **_: maintenance.selected.copy()) + monkeypatch.setattr(lifecycle.node_health, "inspect_unit", lambda *_: None) + monkeypatch.setattr(lifecycle.proxy_install, "check_binding", lambda *_: None) + monkeypatch.setattr( + maintenance, + "unit", + lambda _: { + "ActiveState": "active" if state["active"] else "inactive", + "SubState": "running", + "MainPID": "42" if state["active"] else "0", + }, + ) + monkeypatch.setattr( + maintenance, + "save", + lambda phase: saves.append((phase, [x.copy() for x in maintenance.record["events"]])), + ) + + def systemctl(*args): + assert args == ("--no-block", "start", "kf-proxy.service") + state["active"] = True + return "" + + monkeypatch.setattr(lifecycle.service_install, "systemctl", systemctl) + return maintenance, clock, saves + + +@pytest.mark.parametrize( + "group,active", [(True, False), (False, False), (True, True), (False, True)] +) +def test_proxy_process_is_not_confirmed_until_actual_https_probe_succeeds( + monkeypatch, group, active +): + maintenance, clock, saves = setup_maintenance(monkeypatch, active=active) + probes = [] + + def probe(selected, *, timeout): + assert all(not x["confirmed"] for x in maintenance.record["events"]) + probes.append(timeout) + if len(probes) == 1: + raise proxy_health.HTTPSNotReady("launcher has not served HTTPS yet") + return READY.copy() + + monkeypatch.setattr(proxy_health, "observe", probe) + if group: + maintenance.start_group(("proxy",)) + else: + maintenance.change("proxy", "start") + assert probes == [5.0, 5.0] and clock[0] == 100.5 + if not active: + assert saves[0][1][0]["confirmed"] is False + assert saves[-1][1][0]["confirmed"] is True + + +@pytest.mark.parametrize( + "error", [ssl.SSLCertVerificationError("bad certificate"), DeliveryError("changed TLS binding")] +) +def test_proxy_security_failure_is_fatal_without_wait_or_confirmation(monkeypatch, error): + maintenance, clock, _ = setup_maintenance(monkeypatch) + calls = [] + + def probe(*_, **__): + calls.append(1) + raise error + + monkeypatch.setattr(proxy_health, "observe", probe) + with pytest.raises(type(error)): + maintenance.start_group(("proxy",)) + assert calls == [1] and clock[0] == 100 + assert maintenance.record["events"] == [ + {"unit": "proxy", "operation": "start", "confirmed": False} + ] + + +@pytest.mark.parametrize("late_ready", [False, True]) +def test_proxy_wait_cannot_restart_or_succeed_after_original_deadline(monkeypatch, late_ready): + maintenance, clock, _ = setup_maintenance(monkeypatch) + probes = [] + + def unavailable(*_, timeout): + probes.append(timeout) + clock[0] += 131 + if late_ready: + return READY.copy() + raise proxy_health.HTTPSNotReady("still unavailable") + + monkeypatch.setattr(proxy_health, "observe", unavailable) + with pytest.raises(DeliveryError, match="bounded HTTPS startup unconfirmed"): + maintenance.start_group(("proxy",)) + assert probes == [5.0] + assert maintenance.record["events"][0]["confirmed"] is False + + +@pytest.mark.parametrize( + "error", [ConnectionRefusedError("not listening"), TimeoutError("not ready")] +) +def test_proxy_probe_labels_only_transient_transport_failure(monkeypatch, error): + def unavailable(*_, **__): + raise error + + monkeypatch.setattr(proxy_health, "_observe", unavailable) + with pytest.raises(proxy_health.HTTPSNotReady): + proxy_health.observe({}) + + +@pytest.mark.parametrize("timeout", [0, -1, 5.01, True, float("nan")]) +def test_proxy_probe_rejects_unbounded_timeout(timeout): + with pytest.raises(DeliveryError, match="timeout exceeds bound"): + proxy_health.observe({}, timeout=timeout) diff --git a/scripts/release/README.md b/scripts/release/README.md index 88bb0c2..04f0346 100644 --- a/scripts/release/README.md +++ b/scripts/release/README.md @@ -577,6 +577,14 @@ checks. Each ordinary helper has one CPU within the node's two-CPU ceiling; its memory, process and privilege limits remain fixed. These operator budgets do not change student execution budgets. Startup opens admission only after all required readiness queries succeed. +The proxy's `Type=exec` state only confirms its launcher is running. Lifecycle +startup and resume confirm the real pinned HTTPS/API response within the existing +130-second service-confirmation deadline. Only connection refusal, timeouts and +upstream 502/503/504 observations retry within that same deadline. Certificate, +hostname, configuration and response-validation failures still refuse startup; +ordinary health queries remain single observations. No service, node or student +resource limit is increased. + Database authority queries use the fixed local TCP/Unix endpoints with TLS disabled and a five-second connection deadline. SQL remains read-only with a two-second diff --git a/scripts/release/lifecycle.py b/scripts/release/lifecycle.py index 78a4ea2..5ee2544 100644 --- a/scripts/release/lifecycle.py +++ b/scripts/release/lifecycle.py @@ -22,6 +22,7 @@ from . import ( operation_proof, operator_units, preflight, + proxy_health, proxy_install, proxy_service, service_install, @@ -372,6 +373,8 @@ class Maintenance: values = self.unit(name) if operation == "start" and values["ActiveState"] == "active": node_health.inspect_unit(name, self.selected) + if name == "proxy": + self._wait_for_proxy(time.monotonic() + 130) return if operation == "stop" and values["ActiveState"] == "inactive" and values["MainPID"] == "0": self.empty_cgroup(name) @@ -396,6 +399,8 @@ class Maintenance: and values["SubState"] == "running" ): node_health.inspect_unit(name, self.selected) + if name == "proxy": + self._wait_for_proxy(deadline) break if ( operation == "stop" @@ -410,6 +415,40 @@ class Maintenance: self.record["events"][-1]["confirmed"] = True self.save(operation + "_confirmed") + def _wait_for_proxy(self, deadline: float) -> None: + """Type=exec is process startup; confirm actual HTTPS before admission.""" + while True: + if time.monotonic() >= deadline: + raise DeliveryError("bounded HTTPS startup unconfirmed") + if install.selection(check_configuration=False) != self.selected: + raise DeliveryError("selected binding changed during maintenance") + # Invalid selected TLS/configuration is fatal, never a retry reason. + proxy_install.check_binding( + self.selected["release_id"], self.selected["configuration_id"] + ) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise DeliveryError("bounded HTTPS startup unconfirmed") + try: + ready = proxy_health.observe(self.selected, timeout=min(5.0, remaining)) + except proxy_health.HTTPSNotReady as error: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise DeliveryError("bounded HTTPS startup unconfirmed") from error + time.sleep(min(0.5, remaining)) + continue + if time.monotonic() >= deadline: + raise DeliveryError("bounded HTTPS startup unconfirmed") + if install.selection(check_configuration=False) != self.selected: + raise DeliveryError("selected binding changed during maintenance") + if ready != { + "reachable": True, + "pinned_certificate_and_hostname_verified": True, + "public_browser_trust_verified": False, + }: + raise DeliveryError("HTTPS startup proof differs") + return + def start_group(self, names: tuple[str, ...]) -> None: """Dispatch one fixed dependency stage; confirm each real unit separately.""" if names not in START_GROUPS: @@ -424,6 +463,8 @@ class Maintenance: values = self.unit(name) if values["ActiveState"] == "active": node_health.inspect_unit(name, self.selected) + if name == "proxy": + self._wait_for_proxy(time.monotonic() + 130) else: inactive.append(name) pending = {} @@ -448,6 +489,8 @@ class Maintenance: raise DeliveryError("bounded lifecycle operation unconfirmed") if values["ActiveState"] == "active" and values["SubState"] == "running": node_health.inspect_unit(name, self.selected) + if name == "proxy": + self._wait_for_proxy(deadline) event["confirmed"] = True self.save("start_confirmed") del pending[name] diff --git a/scripts/release/proxy_health.py b/scripts/release/proxy_health.py index 46d1c1c..87daf8d 100644 --- a/scripts/release/proxy_health.py +++ b/scripts/release/proxy_health.py @@ -10,7 +10,20 @@ from . import proxy_service from .policy import DeliveryError, read_json -def observe(selected: dict) -> dict: +class HTTPSNotReady(DeliveryError): + """A bounded transport or upstream startup observation is unavailable.""" + + +def observe(selected: dict, *, timeout: float = 5.0) -> dict: + if type(timeout) not in (int, float) or not 0 < timeout <= 5: + raise DeliveryError("HTTPS observation timeout exceeds bound") + try: + return _observe(selected, timeout=timeout) + except (ConnectionRefusedError, TimeoutError) as error: + raise HTTPSNotReady("HTTPS listener or API is not ready") from error + + +def _observe(selected: dict, *, timeout: float) -> dict: directory = proxy_service.batch(selected) spec = read_json(directory / "launch.json", 4096) host = urlsplit(spec["origin"]).hostname @@ -22,9 +35,9 @@ def observe(selected: dict) -> dict: ctx.load_verify_locations(cafile=certificate) ctx.verify_flags |= ssl.VERIFY_X509_PARTIAL_CHAIN ctx.minimum_version = ssl.TLSVersion.TLSv1_2 - with socket.create_connection(("127.0.0.1", 443), timeout=5) as raw: + with socket.create_connection(("127.0.0.1", 443), timeout=timeout) as raw: with ctx.wrap_socket(raw, server_hostname=host) as client: - client.settimeout(5) + client.settimeout(timeout) der = client.getpeercert(binary_form=True) pem = certificate.read_text() leaf = pem[: pem.index("-----END CERTIFICATE-----") + len("-----END CERTIFICATE-----")] @@ -41,6 +54,8 @@ def observe(selected: dict) -> dict: response = http.client.HTTPResponse(client) try: response.begin() + if response.status in (502, 503, 504): + raise HTTPSNotReady("HTTPS API upstream is not ready") body = response.read(4097) if response.status != 200 or len(body) > 4096: raise DeliveryError("HTTPS health response exceeds bound or is unavailable")