diff --git a/backend/tests/test_delivery.py b/backend/tests/test_delivery.py index e7479fb..e6ed822 100644 --- a/backend/tests/test_delivery.py +++ b/backend/tests/test_delivery.py @@ -973,6 +973,44 @@ def test_maintenance_receipt_cannot_silently_adopt_unknown_history_or_release_pr lifecycle.validate_record(value) +def test_confirmed_drain_can_continue_to_stop_without_querying_stopped_owners(monkeypatch): + from types import SimpleNamespace + + selected = {"release_id": "kf-" + "b" * 12, "configuration_id": "cfg-" + "c" * 24} + previous = { + "schema_version": 1, + "source_sha": "b" * 40, + "configuration_id": selected["configuration_id"], + "action": "drain", + "phase": "complete", + "success": True, + "events": [], + "root_cleanup_confirmed": True, + "boot_id": "11111111-1111-4111-8111-111111111111", + "drain_projection": empty_drain_projection(), + } + maintenance = object.__new__(lifecycle.Maintenance) + maintenance.selected = selected + maintenance.record = {**previous, "action": "stop", "root_cleanup_confirmed": False} + gate = SimpleNamespace(**selected, state="quiescent") + monkeypatch.setattr(lifecycle.availability, "read", lambda: gate) + observed = [] + monkeypatch.setattr( + maintenance, "unit", lambda name: {"ActiveState": "inactive", "MainPID": "0"} + ) + monkeypatch.setattr(maintenance, "empty_cgroup", lambda name: observed.append(name)) + monkeypatch.setattr(maintenance, "root_stopped", lambda: observed.append("root-proof")) + assert maintenance.reuse_drained(previous) is True + assert observed == [*lifecycle.APPS, "root-proof"] + assert maintenance.record["drain_projection"] == previous["drain_projection"] + observed.clear() + with pytest.raises(policy.DeliveryError): + maintenance.reuse_drained({**previous, "boot_id": "22222222-2222-4222-8222-222222222222"}) + assert observed == [] + gate.state = "draining" + assert maintenance.reuse_drained(previous) is False + + @pytest.mark.parametrize( "field,value", [ diff --git a/scripts/release/README.md b/scripts/release/README.md index 1995952..c0914c3 100644 --- a/scripts/release/README.md +++ b/scripts/release/README.md @@ -384,8 +384,12 @@ and hash-checks existing source/case bytes with a 64-KiB buffer. It creates no storage directory or lock and changes no metadata, grade, session or grant. Its output is private and does not itself prove that writers have stopped. -After the confirmed drain, stop the selected database and broker and verify all -their processes have exited. `python3.13 -m scripts.release.backup capture +After collecting the inventory, run `lifecycle --action stop`. For an already +quiescent node it reuses only this boot's successful selected owner-derived drain +receipt, rechecks every application cgroup and the Root task cleanup, then stops +the selected database and broker. It does not query owners already stopped or +adopt a stale, failed or unproven cut. Verify all their processes have exited. +`python3.13 -m scripts.release.backup capture --backup-id bk-<32-lowercase-hex> --references PRIVATE_INVENTORY --references-sha256 SHA` launches one fixed Root backup operator. It requires the same selected release/configuration, this boot's successful owner-derived cut, diff --git a/scripts/release/lifecycle.py b/scripts/release/lifecycle.py index 2b086e3..fb27b0b 100644 --- a/scripts/release/lifecycle.py +++ b/scripts/release/lifecycle.py @@ -430,12 +430,46 @@ class Maintenance: final = self.snapshot() if not projection_empty(final): raise DeliveryError("work changed during final stop; quiescence unconfirmed") + self.record["drain_projection"] = final self.change("coordinator", "stop") self.change("sandbox-factory", "stop") self.root_stopped() availability.write(self.selected, "quiescent") self.save("quiescent") + def reuse_drained(self, previous: dict | None) -> bool: + """Continue a confirmed drain without querying owners already stopped.""" + gate = availability.read() + if gate is None or gate.state != "quiescent": + return False + if previous is None: + raise DeliveryError("quiescent node has no confirmed drain receipt") + validate_record(previous) + if ( + gate.release_id != self.selected["release_id"] + or gate.configuration_id != self.selected["configuration_id"] + or any( + previous[key] != self.record[key] + for key in ("source_sha", "configuration_id", "boot_id") + ) + or previous["action"] not in {"drain", "stop"} + or previous["phase"] != "complete" + or previous["success"] is not True + or previous["root_cleanup_confirmed"] is not True + or any(event["confirmed"] is not True for event in previous["events"]) + or "drain_projection" not in previous + or not projection_empty(previous["drain_projection"]) + ): + raise DeliveryError("prior drain is not this boot's selected owner-confirmed cut") + for name in APPS: + values = self.unit(name) + if values["ActiveState"] != "inactive" or values["MainPID"] != "0": + raise DeliveryError("prior drain has a running application service") + self.empty_cgroup(name) + self.record["drain_projection"] = previous["drain_projection"] + self.root_stopped() + return True + def start(self, *, resume: bool) -> None: if not resume: for name in APPS: @@ -508,14 +542,16 @@ def run(action: str) -> dict: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) # Refuse unknown history before publishing availability, including # resume where every known service could already be running. + previous = None if LAST.exists() or LAST.is_symlink(): preflight.protected(LAST) configuration.private(LAST) - validate_record(read_json(LAST, 32768)) + previous = validate_record(read_json(LAST, 32768)) maintenance = Maintenance(selected, paths, payload, action) try: if action in {"drain", "stop", "restart"}: - maintenance.drain() + if not maintenance.reuse_drained(previous): + maintenance.drain() if action == "stop": for name in ("rabbitmq", "postgres"): maintenance.change(name, "stop")