From 2cae75920d662abbb80eca91dd9d99d90453bec0 Mon Sep 17 00:00:00 2001 From: kronecker <2026692687@qq.com> Date: Fri, 16 Oct 2026 13:38:49 +0800 Subject: [PATCH] fix(release): honor factory preparation startup deadline --- backend/tests/test_delivery.py | 38 ++++++++++++++++++++++++++++++ scripts/release/factory_service.py | 17 ++++++++++--- scripts/release/lifecycle.py | 16 +++++++++++-- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/backend/tests/test_delivery.py b/backend/tests/test_delivery.py index 0fe4c85..91b0287 100644 --- a/backend/tests/test_delivery.py +++ b/backend/tests/test_delivery.py @@ -486,6 +486,44 @@ def test_startup_cannot_launch_services_when_recovery_admission_is_refused(monke assert calls == [] +@pytest.mark.parametrize( + ("name", "ready_after", "succeeds"), + [("sandbox-factory", 131, True), ("sandbox-factory", 1811, False), ("api", 131, False)], +) +def test_startup_waits_for_declared_factory_preparation_without_extending_other_services( + monkeypatch, name, ready_after, succeeds +): + maintenance = object.__new__(lifecycle.Maintenance) + maintenance.selected = {"release_id": "sd-" + "a" * 12, "configuration_id": "cfg-" + "b" * 24} + maintenance.record = {"events": [], "root_cleanup_confirmed": True} + clock = [0.0] + confirmed = [] + monkeypatch.setattr(lifecycle.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr(lifecycle.time, "sleep", lambda _: clock.__setitem__(0, clock[0] + 1)) + monkeypatch.setattr(lifecycle.install, "selection", lambda **_: maintenance.selected) + monkeypatch.setattr(lifecycle.service_install, "systemctl", lambda *args: "") + monkeypatch.setattr(maintenance, "save", lambda phase: None) + monkeypatch.setattr( + maintenance, + "unit", + lambda _: { + "ActiveState": "active" if clock[0] >= ready_after else "activating", + "SubState": "running" if clock[0] >= ready_after else "start-pre", + "MainPID": "123" if clock[0] >= ready_after else "0", + }, + ) + monkeypatch.setattr(lifecycle.node_health, "inspect_unit", lambda *args: confirmed.append(args)) + if succeeds: + maintenance.change(name, "start") + assert maintenance.record["events"][-1]["confirmed"] is True + assert confirmed == [(name, maintenance.selected)] + else: + with pytest.raises(policy.DeliveryError, match="bounded lifecycle operation unconfirmed"): + maintenance.change(name, "start") + assert maintenance.record["events"][-1]["confirmed"] is False + assert confirmed == [] + + def test_restoration_keeps_old_runtime_configuration_and_operations_inactive(): directory = Path("/var/lib/kf-onlinejudge/recovery/rr-" + "1" * 32) for domain in restore.HISTORICAL: diff --git a/scripts/release/factory_service.py b/scripts/release/factory_service.py index becbd4f..d505bc7 100644 --- a/scripts/release/factory_service.py +++ b/scripts/release/factory_service.py @@ -14,6 +14,7 @@ from .policy import DeliveryError, canonical, digest, fields, read_json, sha256 UNIT = "kf-sandbox-factory.service" UNITS = Path("/etc/kf-judge/units") +START_TIMEOUT_SECONDS = 1800 def validate_receipt(value: dict) -> dict: @@ -143,7 +144,7 @@ def render(receipt: dict) -> bytes: "LimitFSIZE=64M", "LimitNOFILE=256", "LimitSTACK=16M", - "TimeoutStartSec=1800", + f"TimeoutStartSec={START_TIMEOUT_SECONDS}", "RuntimeMaxSec=3700", "TimeoutStopSec=40", "KillMode=mixed", @@ -164,7 +165,12 @@ def generate(output: Path) -> dict: raise DeliveryError("Root factory composition requires trusted operator") selected, original = service_install.current_receipt() payload = install.RELEASES / selected["release_id"] / "payload" - for name in ("factory_service.py", "services.py", "service_profiles.py", "layout.py"): + for name in ( + "factory_service.py", + "services.py", + "service_profiles.py", + "layout.py", + ): installed = payload / "delivery/scripts/release" / name preflight.protected(installed) if digest(installed) != digest(Path(__file__).parent / name): @@ -211,7 +217,12 @@ def install_unit(source: Path, expected: str) -> dict: ): raise DeliveryError("factory unit does not belong to selected configuration") payload = install.RELEASES / selected["release_id"] / "payload" - for name in ("factory_service.py", "services.py", "service_profiles.py", "layout.py"): + for name in ( + "factory_service.py", + "services.py", + "service_profiles.py", + "layout.py", + ): shipped = payload / "delivery/scripts/release" / name preflight.protected(shipped) if digest(shipped) != digest(Path(__file__).parent / name): diff --git a/scripts/release/lifecycle.py b/scripts/release/lifecycle.py index 5dc35b8..f20c121 100644 --- a/scripts/release/lifecycle.py +++ b/scripts/release/lifecycle.py @@ -30,7 +30,15 @@ from .policy import DeliveryError, canonical, digest, fields, read_json, sha256 STATE = Path("/var/lib/kf-onlinejudge/operations") LAST = STATE / "maintenance.last.json" -APPS = ("proxy", "api", "outbox", "worker", "problem-import", "coordinator", "sandbox-factory") +APPS = ( + "proxy", + "api", + "outbox", + "worker", + "problem-import", + "coordinator", + "sandbox-factory", +) DEPENDENCIES = ("postgres", "rabbitmq") START_ORDER = ( "sandbox-factory", @@ -337,7 +345,11 @@ class Maintenance: self.record["events"].append({"unit": name, "operation": operation, "confirmed": False}) self.save("dispatching_" + operation) service_install.systemctl("--no-block", operation, "kf-" + name + ".service") - deadline = time.monotonic() + (130 if operation == "start" else 50) + # Root preparation validates the protected image registry before exec. + # Its declared unit timeout also applies here; the enclosing maintenance + # helper still enforces its independent overall runtime ceiling. + start_seconds = factory_service.START_TIMEOUT_SECONDS if name == "sandbox-factory" else 120 + deadline = time.monotonic() + (start_seconds + 10 if operation == "start" else 50) while True: if install.selection(check_configuration=False) != self.selected: raise DeliveryError("selected binding changed during maintenance")