fix(release): honor factory preparation startup deadline

backend/tests/test_delivery.py

@@ -486,6 +486,44 @@ def test_startup_cannot_launch_services_when_recovery_admission_is_refused(monke
     assert calls == []
 
 
+@pytest.mark.parametrize(
+    ("name", "ready_after", "succeeds"),
+    [("sandbox-factory", 131, True), ("sandbox-factory", 1811, False), ("api", 131, False)],
+)
+def test_startup_waits_for_declared_factory_preparation_without_extending_other_services(
+    monkeypatch, name, ready_after, succeeds
+):
+    maintenance = object.__new__(lifecycle.Maintenance)
+    maintenance.selected = {"release_id": "sd-" + "a" * 12, "configuration_id": "cfg-" + "b" * 24}
+    maintenance.record = {"events": [], "root_cleanup_confirmed": True}
+    clock = [0.0]
+    confirmed = []
+    monkeypatch.setattr(lifecycle.time, "monotonic", lambda: clock[0])
+    monkeypatch.setattr(lifecycle.time, "sleep", lambda _: clock.__setitem__(0, clock[0] + 1))
+    monkeypatch.setattr(lifecycle.install, "selection", lambda **_: maintenance.selected)
+    monkeypatch.setattr(lifecycle.service_install, "systemctl", lambda *args: "")
+    monkeypatch.setattr(maintenance, "save", lambda phase: None)
+    monkeypatch.setattr(
+        maintenance,
+        "unit",
+        lambda _: {
+            "ActiveState": "active" if clock[0] >= ready_after else "activating",
+            "SubState": "running" if clock[0] >= ready_after else "start-pre",
+            "MainPID": "123" if clock[0] >= ready_after else "0",
+        },
+    )
+    monkeypatch.setattr(lifecycle.node_health, "inspect_unit", lambda *args: confirmed.append(args))
+    if succeeds:
+        maintenance.change(name, "start")
+        assert maintenance.record["events"][-1]["confirmed"] is True
+        assert confirmed == [(name, maintenance.selected)]
+    else:
+        with pytest.raises(policy.DeliveryError, match="bounded lifecycle operation unconfirmed"):
+            maintenance.change(name, "start")
+        assert maintenance.record["events"][-1]["confirmed"] is False
+        assert confirmed == []
+
+
 def test_restoration_keeps_old_runtime_configuration_and_operations_inactive():
     directory = Path("/var/lib/kf-onlinejudge/recovery/rr-" + "1" * 32)
     for domain in restore.HISTORICAL:

scripts/release/factory_service.py

@@ -14,6 +14,7 @@ from .policy import DeliveryError, canonical, digest, fields, read_json, sha256
 
 UNIT = "kf-sandbox-factory.service"
 UNITS = Path("/etc/kf-judge/units")
+START_TIMEOUT_SECONDS = 1800
 
 
 def validate_receipt(value: dict) -> dict:
@@ -143,7 +144,7 @@ def render(receipt: dict) -> bytes:
         "LimitFSIZE=64M",
         "LimitNOFILE=256",
         "LimitSTACK=16M",
-        "TimeoutStartSec=1800",
+        f"TimeoutStartSec={START_TIMEOUT_SECONDS}",
         "RuntimeMaxSec=3700",
         "TimeoutStopSec=40",
         "KillMode=mixed",
@@ -164,7 +165,12 @@ def generate(output: Path) -> dict:
         raise DeliveryError("Root factory composition requires trusted operator")
     selected, original = service_install.current_receipt()
     payload = install.RELEASES / selected["release_id"] / "payload"
-    for name in ("factory_service.py", "services.py", "service_profiles.py", "layout.py"):
+    for name in (
+        "factory_service.py",
+        "services.py",
+        "service_profiles.py",
+        "layout.py",
+    ):
         installed = payload / "delivery/scripts/release" / name
         preflight.protected(installed)
         if digest(installed) != digest(Path(__file__).parent / name):
@@ -211,7 +217,12 @@ def install_unit(source: Path, expected: str) -> dict:
     ):
         raise DeliveryError("factory unit does not belong to selected configuration")
     payload = install.RELEASES / selected["release_id"] / "payload"
-    for name in ("factory_service.py", "services.py", "service_profiles.py", "layout.py"):
+    for name in (
+        "factory_service.py",
+        "services.py",
+        "service_profiles.py",
+        "layout.py",
+    ):
         shipped = payload / "delivery/scripts/release" / name
         preflight.protected(shipped)
         if digest(shipped) != digest(Path(__file__).parent / name):

scripts/release/lifecycle.py

@@ -30,7 +30,15 @@ from .policy import DeliveryError, canonical, digest, fields, read_json, sha256
 
 STATE = Path("/var/lib/kf-onlinejudge/operations")
 LAST = STATE / "maintenance.last.json"
-APPS = ("proxy", "api", "outbox", "worker", "problem-import", "coordinator", "sandbox-factory")
+APPS = (
+    "proxy",
+    "api",
+    "outbox",
+    "worker",
+    "problem-import",
+    "coordinator",
+    "sandbox-factory",
+)
 DEPENDENCIES = ("postgres", "rabbitmq")
 START_ORDER = (
     "sandbox-factory",
@@ -337,7 +345,11 @@ class Maintenance:
         self.record["events"].append({"unit": name, "operation": operation, "confirmed": False})
         self.save("dispatching_" + operation)
         service_install.systemctl("--no-block", operation, "kf-" + name + ".service")
-        deadline = time.monotonic() + (130 if operation == "start" else 50)
+        # Root preparation validates the protected image registry before exec.
+        # Its declared unit timeout also applies here; the enclosing maintenance
+        # helper still enforces its independent overall runtime ceiling.
+        start_seconds = factory_service.START_TIMEOUT_SECONDS if name == "sandbox-factory" else 120
+        deadline = time.monotonic() + (start_seconds + 10 if operation == "start" else 50)
         while True:
             if install.selection(check_configuration=False) != self.selected:
                 raise DeliveryError("selected binding changed during maintenance")