diff --git a/backend/tests/test_delivery.py b/backend/tests/test_delivery.py index 7d62f5f..e7479fb 100644 --- a/backend/tests/test_delivery.py +++ b/backend/tests/test_delivery.py @@ -30,6 +30,31 @@ journal = importlib.import_module("scripts.release.journal") proxy_runtime = importlib.import_module("scripts.release.proxy_runtime") proxy_service = importlib.import_module("scripts.release.proxy_service") proxy_guard = importlib.import_module("scripts.release.proxy_guard") +backup = importlib.import_module("scripts.release.backup") + + +def test_backup_retention_preserves_a_newer_unexpired_recovery_copy(): + expired = { + "backup_id": "bk-" + "1" * 32, + "created_unix_seconds": 100, + "expires_unix_seconds": 100 + 7 * 86400, + } + retained = { + "backup_id": "bk-" + "2" * 32, + "created_unix_seconds": 200, + "expires_unix_seconds": 200 + 7 * 86400, + } + now = expired["expires_unix_seconds"] + backup.retention_eligible(expired, retained, now) + for unavailable in ( + expired, + {**retained, "expires_unix_seconds": now}, + {**retained, "created_unix_seconds": now + 1}, + ): + with pytest.raises(policy.DeliveryError): + backup.retention_eligible(expired, unavailable, now) + with pytest.raises(policy.DeliveryError): + backup.retention_eligible(expired, retained, now - 1) def test_delivery_authorities_and_layout(): diff --git a/scripts/release/README.md b/scripts/release/README.md index 61ea9fa..1995952 100644 --- a/scripts/release/README.md +++ b/scripts/release/README.md @@ -411,8 +411,15 @@ with a seven-day declared retention period. Capacity refusal does not silently delete an old/unknown backup. `backup verify --backup-id ID --manifest-sha256 SHA` checks an explicitly supplied private operator receipt and every private member; it performs no restoration or secret/epoch reuse. Actual -retention pruning, controlled recovery into a clean node and measured recovery -time require their separate implementation and independent VM acceptance. +pruning uses `backup prune --backup-id EXPIRED_ID --manifest-sha256 SHA +--keep-backup-id NEWER_ID --keep-manifest-sha256 SHA`. Both private inventories +must verify completely; the named target must be expired and the preserved copy +must be newer and unexpired. It removes only the known inventory under the +maintenance lock, never the last verified copy, an unknown tree or a partial +backup. Verification and pruning use the same bounded operator as capture. +Controlled recovery into a clean node and measured recovery time require their +separate implementation and independent VM acceptance. Capture, expiry pruning +and interruption behavior also still require actual VM acceptance. Backup data and manifests contain private credentials/source/testcase data and must stay outside Git and CI artifacts. Parsing, ordinary adapter tests or a manifest digest are not cold-backup/recovery acceptance evidence. diff --git a/scripts/release/backup.py b/scripts/release/backup.py index fc052b8..458c7ca 100644 --- a/scripts/release/backup.py +++ b/scripts/release/backup.py @@ -84,8 +84,12 @@ def stopped(selected: dict, cut: dict) -> None: raise DeliveryError("cold backup has an active required service") if props.get("ControlGroup"): group = Path("/sys/fs/cgroup") / props["ControlGroup"].lstrip("/") - if group.exists() and (group / "cgroup.procs").read_text().strip(): - raise DeliveryError("cold backup service processes remain") + if group.exists(): + events = dict( + line.split() for line in (group / "cgroup.events").read_text().splitlines() + ) + if events.get("populated") != "0": + raise DeliveryError("cold backup service descendant processes remain") for name in ("tasks", "identities", "captures"): path = Path("/var/lib/kf-judge") / name if path.exists() and next(path.iterdir(), None) is not None: @@ -109,7 +113,7 @@ def stopped(selected: dict, cut: dict) -> None: raise DeliveryError("PostgreSQL did not confirm clean cold shutdown") -def volume() -> None: +def volume(*, permit_full: bool = False) -> None: preflight.protected(ROOT, directory=True) if stat.S_IMODE(ROOT.stat().st_mode) != 0o700: raise DeliveryError("private preprovisioned backup volume required") @@ -130,7 +134,11 @@ def volume() -> None: or not CAPACITY * 9 // 10 <= fs.f_frsize * fs.f_blocks <= CAPACITY ): raise DeliveryError("backup volume limits or mount isolation differ") - members = list(ROOT.iterdir()) + members = [] + for member in ROOT.iterdir(): + members.append(member) + if len(members) > 3: + raise DeliveryError("private backup directory count exceeds bound") snapshots = [] for p in members: preflight.protected(p, directory=p.is_dir()) @@ -140,7 +148,7 @@ def volume() -> None: if not p.is_dir() or stat.S_IMODE(p.stat().st_mode) != 0o700: raise DeliveryError("unknown private backup volume member") snapshots.append(p) - if len(snapshots) >= 2: + if len(snapshots) > 2 or (len(snapshots) == 2 and not permit_full): raise DeliveryError( "two-backup retention capacity reached; explicit verified pruning required" ) @@ -286,51 +294,20 @@ def capture(backup_id: str, references_file: Path, references_sha: str) -> dict: < before.st_size + 64 * 1024**2 ): raise DeliveryError("private backup capacity exhausted") - fd = os.open(p, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC) h = hashlib.sha256() - output = os.open( - destination / member, - os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, - 0o600, - ) - with ( - os.fdopen(fd, "rb") as incoming, - os.fdopen(output, "wb") as outgoing, - ): - observed = os.fstat(incoming.fileno()) - if any( - getattr(before, k) != getattr(observed, k) - for k in ( - "st_dev", - "st_ino", - "st_size", - "st_mtime_ns", - "st_ctime_ns", - ) - ): - raise DeliveryError("backup source changed before capture") - count = 0 - while block := incoming.read(65536): - count += len(block) - if count > before.st_size or time.monotonic() >= deadline: - raise DeliveryError("backup source or deadline changed") - h.update(block) - outgoing.write(block) - after = os.fstat(incoming.fileno()) - if count != before.st_size or any( - getattr(before, k) != getattr(after, k) - for k in ( - "st_dev", - "st_ino", - "st_size", - "st_mtime_ns", - "st_ctime_ns", - ) - ): - raise DeliveryError("backup source changed during capture") - outgoing.flush() - os.fsync(outgoing.fileno()) - os.fchmod(outgoing.fileno(), 0o600) + with os.fdopen( + os.open(p, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC), + "rb", + ) as incoming: + with os.fdopen( + os.open( + destination / member, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, + ), + "wb", + ) as outgoing: + copy_file(incoming, outgoing, before, deadline, h) entry["sha256"] = h.hexdigest() files.append(entry) install.sync_directory(destination) @@ -382,114 +359,245 @@ def capture(backup_id: str, references_file: Path, references_sha: str) -> dict: os.close(descriptor) +def copy_file(incoming, outgoing, before, deadline: float, h) -> None: + keys = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + observed = os.fstat(incoming.fileno()) + if any(getattr(before, key) != getattr(observed, key) for key in keys): + raise DeliveryError("backup source changed before capture") + count = 0 + while block := incoming.read(65536): + count += len(block) + if count > before.st_size or time.monotonic() >= deadline: + raise DeliveryError("backup source or deadline changed") + h.update(block) + outgoing.write(block) + after = os.fstat(incoming.fileno()) + if count != before.st_size or any(getattr(before, key) != getattr(after, key) for key in keys): + raise DeliveryError("backup source changed during capture") + outgoing.flush() + os.fsync(outgoing.fileno()) + os.fchmod(outgoing.fileno(), 0o600) + + +def retention_eligible(expired: dict, retained: dict, now: int) -> None: + """An explicit expired snapshot never removes the last verified recovery copy.""" + if ( + expired["backup_id"] == retained["backup_id"] + or expired["expires_unix_seconds"] > now + or retained["created_unix_seconds"] > now + or retained["created_unix_seconds"] <= expired["created_unix_seconds"] + or retained["expires_unix_seconds"] <= now + ): + raise DeliveryError("pruning requires an expired snapshot and a newer unexpired copy") + + +def prune(backup_id: str, expected: str, keep_id: str, keep_expected: str) -> dict: + guarded() + volume(permit_full=True) + target, retained = ROOT / identifier(backup_id), ROOT / identifier(keep_id) + lock = lifecycle.STATE / "maintenance.lock" + configuration.private(lock) + descriptor = os.open(lock, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + deadline = time.monotonic() + 1700 + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + value = backup_manifest.verify(target, expected) + keep = backup_manifest.verify(retained, keep_expected) + if value["backup_id"] != backup_id or keep["backup_id"] != keep_id: + raise DeliveryError("private backup path does not match its receipt") + retention_eligible(value, keep, int(time.time())) + # Remove exactly the already verified inventory, never traverse or delete + # an unknown tree. Any interruption leaves a private partial target. + for item in value["files"]: + if time.monotonic() >= deadline: + raise DeliveryError("private backup pruning deadline") + path = target / "payload" / item["path"] + info = path.lstat() + if ( + not stat.S_ISREG(info.st_mode) + or info.st_nlink != 1 + or (info.st_uid, info.st_gid, stat.S_IMODE(info.st_mode)) != (0, 0, 0o600) + or info.st_size != item["bytes"] + ): + raise DeliveryError("private backup changed before pruning") + path.unlink() + install.sync_directory(path.parent) + for item in sorted( + value["directories"], key=lambda item: item["path"].count("/"), reverse=True + ): + path = target / "payload" / item["path"] + preflight.protected(path, directory=True) + if stat.S_IMODE(path.stat().st_mode) != 0o700: + raise DeliveryError("private backup directory changed before pruning") + path.rmdir() + install.sync_directory(path.parent) + (target / "payload").rmdir() + configuration.private(target / "manifest.json") + if digest(target / "manifest.json") != sha256(expected): + raise DeliveryError("private backup receipt changed before pruning") + (target / "manifest.json").unlink() + target.rmdir() + install.sync_directory(ROOT) + backup_manifest.verify(retained, keep_expected) + return { + "backup_id": backup_id, + "pruned": True, + "retained_backup_id": keep_id, + "retained_manifest_sha256": keep_expected, + "execution_verified": False, + } + finally: + os.close(descriptor) + + +def launch(args) -> dict: + """Only fixed backup actions; no program, identity or budget override.""" + identifier(args.backup_id) + if args.action == "capture": + configuration.private(args.references) + if digest(args.references) != sha256(args.references_sha256): + raise DeliveryError("private artifact inventory receipt differs") + options = [ + "--references", + str(args.references), + "--references-sha256", + args.references_sha256, + ] + else: + sha256(args.manifest_sha256) + options = ["--manifest-sha256", args.manifest_sha256] + if args.action == "prune": + identifier(args.keep_backup_id) + sha256(args.keep_manifest_sha256) + options += [ + "--keep-backup-id", + args.keep_backup_id, + "--keep-manifest-sha256", + args.keep_manifest_sha256, + ] + selected = install.selection() + payload = install.RELEASES / selected["release_id"] / "payload" + current = manager("show", "kf-node-backup.service", "--property=LoadState,ActiveState,MainPID") + if current != {"LoadState": "not-found", "ActiveState": "inactive", "MainPID": "0"}: + raise DeliveryError("unknown or active backup operator") + command = [ + "/usr/bin/systemd-run", + "--quiet", + "--wait", + "--pipe", + "--collect", + "--unit=kf-node-backup.service", + "--slice=kfonlinejudge.slice", + ] + for key, value in { + "User": "root", + "Group": "root", + "MemoryMax": "256M", + "MemorySwapMax": "0", + "TasksMax": "32", + "CPUQuota": "50%", + "RuntimeMaxSec": "1800", + "TimeoutStopSec": "15", + "KillMode": "control-group", + "Restart": "no", + "LimitCORE": "0", + "LimitFSIZE": "256M", + "LimitNOFILE": "256", + "LimitSTACK": "16M", + "UMask": "0077", + "NoNewPrivileges": "yes", + "CapabilityBoundingSet": "CAP_DAC_READ_SEARCH", + "AmbientCapabilities": "CAP_DAC_READ_SEARCH", + "ProtectSystem": "strict", + "ProtectHome": "yes", + "PrivateTmp": "yes", + "PrivateDevices": "yes", + "ProtectControlGroups": "yes", + "RestrictAddressFamilies": "AF_UNIX", + "ReadWritePaths": str(ROOT), + }.items(): + command += ["--property=" + key + "=" + value] + command += [ + "/usr/bin/env", + "-i", + "PATH=/usr/bin:/bin", + "LC_ALL=C.UTF-8", + *python_environment(payload).split(), + str(payload / "python/bin/python3.13"), + "-s", + "-P", + "-m", + "scripts.release.backup", + args.action, + "--inside", + "--backup-id", + args.backup_id, + *options, + ] + result = subprocess.run(command, capture_output=True, timeout=1820, check=False) + if result.returncode or len(result.stdout) > 4096: + raise DeliveryError( + "private backup action failed; source unchanged and partial target retained" + ) + return json.loads(result.stdout) + + +def selected_operator() -> None: + selected = install.selection() + payload = install.RELEASES / selected["release_id"] / "payload" + for name in ("backup.py", "backup_manifest.py"): + if digest(payload / "delivery/scripts/release" / name) != digest( + Path(__file__).parent / name + ): + raise DeliveryError("backup operator differs from selected release") + + def main() -> None: parser = argparse.ArgumentParser(description="Private bounded cold backup operator") sub = parser.add_subparsers(dest="action", required=True) create = sub.add_parser("capture") - create.add_argument("--backup-id", required=True) create.add_argument("--references", type=Path, required=True) create.add_argument("--references-sha256", required=True) - create.add_argument("--inside", action="store_true") check = sub.add_parser("verify") - check.add_argument("--backup-id", required=True) - check.add_argument("--manifest-sha256", required=True) + remove = sub.add_parser("prune") + for command in (create, check, remove): + command.add_argument("--backup-id", required=True) + command.add_argument("--inside", action="store_true", help=argparse.SUPPRESS) + for command in (check, remove): + command.add_argument("--manifest-sha256", required=True) + remove.add_argument("--keep-backup-id", required=True) + remove.add_argument("--keep-manifest-sha256", required=True) args = parser.parse_args() try: if os.getuid() != 0 or os.geteuid() != 0: raise DeliveryError("explicit root backup operator required") - if args.action == "verify": - value = backup_manifest.verify(ROOT / identifier(args.backup_id), args.manifest_sha256) - result = { - "verified": True, - "backup_id": value["backup_id"], - "restored": False, - "execution_verified": False, - } - elif args.inside: - result = capture(args.backup_id, args.references, args.references_sha256) + if not args.inside: + result = launch(args) else: - # Caller provides only an ID/private digest; no program, identity, - # source-domain or resource override is accepted by this launcher. - identifier(args.backup_id) - configuration.private(args.references) - if digest(args.references) != sha256(args.references_sha256): - raise DeliveryError("private artifact inventory receipt differs") - selected = install.selection() - payload = install.RELEASES / selected["release_id"] / "payload" - current = manager( - "show", - "kf-node-backup.service", - "--property=LoadState,ActiveState,MainPID", - ) - if current != { - "LoadState": "not-found", - "ActiveState": "inactive", - "MainPID": "0", - }: - raise DeliveryError("unknown or active backup operator") - command = [ - "/usr/bin/systemd-run", - "--quiet", - "--wait", - "--pipe", - "--collect", - "--unit=kf-node-backup.service", - "--slice=kfonlinejudge.slice", - ] - for key, value in { - "User": "root", - "Group": "root", - "MemoryMax": "256M", - "MemorySwapMax": "0", - "TasksMax": "32", - "CPUQuota": "50%", - "RuntimeMaxSec": "1800", - "TimeoutStopSec": "15", - "KillMode": "control-group", - "Restart": "no", - "LimitCORE": "0", - "LimitFSIZE": "256M", - "LimitNOFILE": "256", - "LimitSTACK": "16M", - "UMask": "0077", - "NoNewPrivileges": "yes", - "CapabilityBoundingSet": "CAP_DAC_READ_SEARCH", - "AmbientCapabilities": "CAP_DAC_READ_SEARCH", - "ProtectSystem": "strict", - "ProtectHome": "yes", - "PrivateTmp": "yes", - "PrivateDevices": "yes", - "ProtectControlGroups": "yes", - "RestrictAddressFamilies": "AF_UNIX", - "ReadWritePaths": str(ROOT), - }.items(): - command += ["--property=" + key + "=" + value] - command += [ - "/usr/bin/env", - "-i", - "PATH=/usr/bin:/bin", - "LC_ALL=C.UTF-8", - *python_environment(payload).split(), - str(payload / "python/bin/python3.13"), - "-s", - "-P", - "-m", - "scripts.release.backup", - "capture", - "--inside", - "--backup-id", - args.backup_id, - "--references", - str(args.references), - "--references-sha256", - args.references_sha256, - ] - result = subprocess.run(command, capture_output=True, timeout=1820, check=False) - if result.returncode or len(result.stdout) > 4096: - raise DeliveryError( - "cold backup capture failed; source remains stopped and partial target retained" + guarded() + selected_operator() + if args.action == "capture": + result = capture(args.backup_id, args.references, args.references_sha256) + elif args.action == "prune": + result = prune( + args.backup_id, + args.manifest_sha256, + args.keep_backup_id, + args.keep_manifest_sha256, ) - result = json.loads(result.stdout) + else: + volume(permit_full=True) + value = backup_manifest.verify( + ROOT / identifier(args.backup_id), args.manifest_sha256 + ) + if value["backup_id"] != args.backup_id: + raise DeliveryError("private backup path does not match its receipt") + result = { + "verified": True, + "backup_id": value["backup_id"], + "restored": False, + "execution_verified": False, + } print(json.dumps(result, sort_keys=True)) except (OSError, ValueError, subprocess.SubprocessError) as error: parser.exit( diff --git a/scripts/release/backup_manifest.py b/scripts/release/backup_manifest.py index 19e29e2..23172ff 100644 --- a/scripts/release/backup_manifest.py +++ b/scripts/release/backup_manifest.py @@ -311,6 +311,8 @@ def verify(root: Path, expected: str) -> dict: found_files, found_dirs = set(), set() payload = root / "payload" preflight.protected(payload, directory=True) + if stat.S_IMODE(payload.stat().st_mode) != 0o700: + raise DeliveryError("private backup payload directory required") for directory, names, files in os.walk(payload, followlinks=False): for member in names + files: p = Path(directory) / member