diff --git a/backend/src/kf_oj/infra/artifacts/backup.py b/backend/src/kf_oj/infra/artifacts/backup.py new file mode 100644 index 0000000..c51f3d1 --- /dev/null +++ b/backend/src/kf_oj/infra/artifacts/backup.py @@ -0,0 +1,126 @@ +"""Read-only private backup inventory, separate from garbage collection and grading.""" + +import argparse +import asyncio +import hashlib +import json +import os +import stat +from pathlib import Path + +from sqlalchemy import select, text +from sqlalchemy.exc import SQLAlchemyError + +from kf_oj.contracts.judging import ArtifactReference +from kf_oj.core.settings import Settings +from kf_oj.infra.artifacts.models import Artifact +from kf_oj.infra.db.session import build_engine, build_session_factory + +MAX_REFERENCES = 65536 + + +def verify_existing(root: Path, reference: ArtifactReference) -> None: + """No mkdir, lock creation or writable descriptor, even for a missing store.""" + root = root.absolute() + if root.resolve(strict=True) != root: + raise ValueError("backup storage path contains links") + directory = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + info = os.fstat(directory) + if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700: + raise ValueError("backup storage root identity differs") + descriptor = os.open( + reference.digest, + os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC, + dir_fd=directory, + ) + with os.fdopen(descriptor, "rb") as source: + before = os.fstat(source.fileno()) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_nlink != 1 + or before.st_uid != os.getuid() + or stat.S_IMODE(before.st_mode) != 0o600 + or before.st_size != reference.bytes + ): + raise ValueError("backup artifact identity or length differs") + digest = hashlib.sha256() + remaining = reference.bytes + while remaining: + block = source.read(min(65536, remaining)) + if not block: + raise ValueError("backup artifact truncated") + digest.update(block) + remaining -= len(block) + after = os.fstat(source.fileno()) + if ( + source.read(1) + or digest.hexdigest() != reference.digest + or any( + getattr(before, key) != getattr(after, key) + for key in ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + ) + ): + raise ValueError("backup artifact changed or digest differs") + finally: + os.close(directory) + + +async def inventory() -> dict: + """Every ready registry object covers all enforced ready-state foreign keys. + + The operator separately proves a stopped-writing cut. This CLI neither + claims that cut nor reads domain ORM models or changes a lifecycle state. + """ + settings = Settings() + if settings.artifact_root is None: + raise ValueError("artifact root required") + engine = build_engine(settings) + try: + sessions = build_session_factory(engine) + async with sessions.begin() as session: + await session.execute(text("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")) + rows = await session.execute( + select(Artifact.digest, Artifact.bytes) + .where(Artifact.state == "ready") + .order_by(Artifact.digest) + .limit(MAX_REFERENCES + 1) + ) + objects = list(rows) + if len(objects) > MAX_REFERENCES: + raise ValueError("backup registry count exceeds bound") + references = [] + for digest, size in objects: + reference = ArtifactReference(digest=digest, bytes=size) + # A 64-KiB buffer checks actual bytes without source/case output. + await asyncio.to_thread(verify_existing, settings.artifact_root, reference) + references.append(reference.model_dump(mode="json")) + return { + "schema_version": 1, + "kind": "kf.private.artifact-backup-inventory", + "read_only": True, + "quiescence_verified": False, + "boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), + "storage_identity": { + "device": settings.artifact_root.stat().st_dev, + "inode": settings.artifact_root.stat().st_ino, + "uid": settings.artifact_root.stat().st_uid, + }, + "objects": references, + } + finally: + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Private read-only artifact backup inventory") + parser.parse_args() + try: + result = asyncio.run(asyncio.wait_for(inventory(), timeout=300)) + print(json.dumps(result, separators=(",", ":"), sort_keys=True)) + except (ValueError, OSError, SQLAlchemyError, TimeoutError): + parser.exit(1, "Backup artifact inventory refused; no metadata or grade was modified.\n") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/test_artifact_backup.py b/backend/tests/test_artifact_backup.py new file mode 100644 index 0000000..f19aa65 --- /dev/null +++ b/backend/tests/test_artifact_backup.py @@ -0,0 +1,24 @@ +"""Trusted adapter checks; real cold PG/role/permission tests remain VM-only.""" + +import hashlib + +from kf_oj.contracts.judging import ArtifactReference +from kf_oj.infra.artifacts.backup import verify_existing + + +def test_backup_verifies_binary_and_empty_objects_without_creating_storage_entries(tmp_path): + tmp_path.chmod(0o700) + values = [b"", b"\0\xff\n" * 50000] + expected = {} + for value in values: + digest = hashlib.sha256(value).hexdigest() + path = tmp_path / digest + path.write_bytes(value) + path.chmod(0o600) + expected[digest] = value + before = {p.name: (p.read_bytes(), p.stat().st_mtime_ns) for p in tmp_path.iterdir()} + for digest, value in expected.items(): + verify_existing(tmp_path, ArtifactReference(digest=digest, bytes=len(value))) + after = {p.name: (p.read_bytes(), p.stat().st_mtime_ns) for p in tmp_path.iterdir()} + assert after == before + assert not (tmp_path / ".lock").exists() diff --git a/scripts/release/README.md b/scripts/release/README.md index b4d85c5..61ea9fa 100644 --- a/scripts/release/README.md +++ b/scripts/release/README.md @@ -374,3 +374,45 @@ it does not substitute for actual browser CA/HTTPS/security acceptance in the VM The exact `/api/v1/health/live` route permits GET/HEAD for this bounded process liveness check. Other health, internal, execution and control routes remain outside the public proxy allowlist; liveness does not attest judging readiness. + +Private cold backups use a separate format from public release bundles. +`python3.13 -m kf_oj.infra.artifacts.backup` is an ordinary read-only artifact +inventory command. Run it with the selected API database/storage identity after +the owner-confirmed drain and while PostgreSQL is still available. It reads every +ready artifact registry object through a repeatable-read, read-only transaction +and hash-checks existing source/case bytes with a 64-KiB buffer. It creates no +storage directory or lock and changes no metadata, grade, session or grant. +Its output is private and does not itself prove that writers have stopped. + +After the confirmed drain, stop the selected database and broker and verify all +their processes have exited. `python3.13 -m scripts.release.backup capture +--backup-id bk-<32-lowercase-hex> --references PRIVATE_INVENTORY +--references-sha256 SHA` launches one fixed Root backup operator. It requires the +same selected release/configuration, this boot's successful owner-derived cut, +quiescent admission, empty task resources, no surviving service identities and +clean PostgreSQL shutdown. The operator has only `CAP_DAC_READ_SEARCH`, no-new- +privileges, 256-MiB memory, zero swap, 32 tasks, 50% CPU, a 256-MiB individual-file +limit and a 30-minute deadline. Only the preprovisioned private backup volume is +writable; this entry does not format/mount a volume or execute captured content. + +Provision a separate Root-owned 0700 ext4 volume of 16 GiB at +`/var/lib/kf-onlinejudge/backups`, with `nosuid,nodev,noexec` and no shared mount. +The capture retains all source domains: the cold PostgreSQL cluster, source and +testcase store, private import staging, all retained release/image versions, +application/dependency/proxy configuration, factory replay history and operation +receipts. Every entry has a bounded digest/size and original identity/mode; +captured copies remain Root-only. Registered source/case references must all +match captured files. Source modification, links, special files, unknown +identities, insufficient space or an incomplete cut are refused. A failed target +is retained for explicit inspection and never published as a successful backup. + +The bound is 7 GiB and 50,000 entries per backup, at most two backup directories, +with a seven-day declared retention period. Capacity refusal does not silently +delete an old/unknown backup. `backup verify --backup-id ID +--manifest-sha256 SHA` checks an explicitly supplied private operator receipt and +every private member; it performs no restoration or secret/epoch reuse. Actual +retention pruning, controlled recovery into a clean node and measured recovery +time require their separate implementation and independent VM acceptance. +Backup data and manifests contain private credentials/source/testcase data and +must stay outside Git and CI artifacts. Parsing, ordinary adapter tests or a +manifest digest are not cold-backup/recovery acceptance evidence. diff --git a/scripts/release/backup.py b/scripts/release/backup.py new file mode 100644 index 0000000..fc052b8 --- /dev/null +++ b/scripts/release/backup.py @@ -0,0 +1,504 @@ +"""Explicit cold private backup; stopped owner proofs precede any data capture.""" + +import argparse +import fcntl +import hashlib +import json +import os +import re +import resource +import stat +import subprocess +import time +from pathlib import Path + +from . import ( + availability, + backup_manifest, + configuration, + install, + lifecycle, + preflight, + service_install, +) +from .policy import DeliveryError, canonical, digest, read_json, sha256 +from .services import python_environment + +ROOT = Path("/var/lib/kf-onlinejudge/backups") +CAPACITY = 16 * 1024**3 +SOURCES = { + "postgres": Path("/var/lib/kf-onlinejudge/postgres/data"), + "artifacts": Path("/var/lib/kf-onlinejudge/artifacts"), + "imports": Path("/var/lib/kf-onlinejudge/imports"), + "app-config": Path("/etc/kf-onlinejudge"), + "judge-config": Path("/etc/kf-judge"), + "factory": Path("/var/lib/kf-judge/factory"), + "operations": lifecycle.STATE, + "releases": install.RELEASES, + "images": install.IMAGES, + "dependencies": Path("/etc/kf-dependencies"), + "proxy": Path("/etc/kf-proxy"), +} + + +def identifier(value: str) -> str: + if not isinstance(value, str) or re.fullmatch(r"bk-[0-9a-f]{32}", value) is None: + raise DeliveryError("explicit private backup ID required") + return value + + +def manager(*args: str) -> dict: + raw = service_install.systemctl(*args) + return dict(line.split("=", 1) for line in raw.splitlines() if "=" in line) + + +def stopped(selected: dict, cut: dict) -> None: + lifecycle.validate_record(cut) + if ( + cut["source_sha"][:12] != selected["release_id"][3:] + or cut["configuration_id"] != selected["configuration_id"] + or cut["boot_id"] != Path("/proc/sys/kernel/random/boot_id").read_text().strip() + or cut["action"] not in {"drain", "stop"} + or cut["phase"] != "complete" + or cut["success"] is not True + or cut["root_cleanup_confirmed"] is not True + or "drain_projection" not in cut + or not lifecycle.projection_empty(cut["drain_projection"]) + ): + raise DeliveryError("cold backup requires this boot owner-confirmed drained cut") + record = availability.read() + if ( + record is None + or record.release_id != selected["release_id"] + or record.configuration_id != selected["configuration_id"] + or record.state != "quiescent" + ): + raise DeliveryError("cold backup admission is not proven quiescent") + for name in [*lifecycle.APPS, *lifecycle.DEPENDENCIES]: + props = manager( + "show", + "kf-" + name + ".service", + "--property=MainPID,ActiveState,ControlGroup", + ) + if props.get("MainPID") != "0" or props.get("ActiveState") != "inactive": + raise DeliveryError("cold backup has an active required service") + if props.get("ControlGroup"): + group = Path("/sys/fs/cgroup") / props["ControlGroup"].lstrip("/") + if group.exists() and (group / "cgroup.procs").read_text().strip(): + raise DeliveryError("cold backup service processes remain") + for name in ("tasks", "identities", "captures"): + path = Path("/var/lib/kf-judge") / name + if path.exists() and next(path.iterdir(), None) is not None: + raise DeliveryError("cold backup task resources remain") + if Path("/var/lib/kf-judge/factory/control.sock").exists(): + raise DeliveryError("cold backup factory socket remains") + processes = [p for p in Path("/proc").iterdir() if p.name.isdecimal()] + if len(processes) > 1024: + raise DeliveryError("cold backup process observation exceeds bound") + for p in processes: + try: + status = (p / "status").read_text() + except FileNotFoundError: + continue + rows = [line.split()[1:] for line in status.splitlines() if line.startswith("Uid:")] + if len(rows) != 1 or len(rows[0]) != 4: + raise DeliveryError("cold backup process identity unavailable") + if any(int(uid) in {1200, 1201, 1202, 1203, 1204, 1205, 100000} for uid in rows[0]): + raise DeliveryError("cold backup unmanaged data writer or task remains") + if (SOURCES["postgres"] / "postmaster.pid").exists(): + raise DeliveryError("PostgreSQL did not confirm clean cold shutdown") + + +def volume() -> None: + preflight.protected(ROOT, directory=True) + if stat.S_IMODE(ROOT.stat().st_mode) != 0o700: + raise DeliveryError("private preprovisioned backup volume required") + rows = [line.split() for line in Path("/proc/self/mountinfo").read_text().splitlines()] + matches = [r for r in rows if r[4] == str(ROOT)] + if len(matches) != 1: + raise DeliveryError("separate hard-sized backup filesystem required") + row = matches[0] + sep = row.index("-") + opts = set(row[5].split(",")) | set(row[sep + 3].split(",")) + fs = os.statvfs(ROOT) + if ( + row[3] != "/" + or row[sep + 1] != "ext4" + or any(r.startswith(("shared:", "master:")) for r in row[6:sep]) + or not {"rw", "nosuid", "nodev", "noexec"} <= opts + or "ro" in opts + or not CAPACITY * 9 // 10 <= fs.f_frsize * fs.f_blocks <= CAPACITY + ): + raise DeliveryError("backup volume limits or mount isolation differ") + members = list(ROOT.iterdir()) + snapshots = [] + for p in members: + preflight.protected(p, directory=p.is_dir()) + if p.name == "lost+found" and p.is_dir() and not list(p.iterdir()): + continue + identifier(p.name) + if not p.is_dir() or stat.S_IMODE(p.stat().st_mode) != 0o700: + raise DeliveryError("unknown private backup volume member") + snapshots.append(p) + if len(snapshots) >= 2: + raise DeliveryError( + "two-backup retention capacity reached; explicit verified pruning required" + ) + + +def guarded() -> None: + if (os.getuid(), os.geteuid(), os.getgid()) != (0, 0, 0) or not set(os.getgroups()) <= {0}: + raise DeliveryError("root backup operator required") + status = dict( + line.split(":", 1) + for line in Path("/proc/self/status").read_text().splitlines() + if ":" in line + ) + if status.get("NoNewPrivs", "").strip() != "1" or any( + int(status.get(k, "-1").strip(), 16) != 4 + for k in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb") + ): + raise DeliveryError("fixed read-search-only backup capability required") + group = preflight.current_cgroup() + if group != Path("/sys/fs/cgroup/kfonlinejudge.slice/kf-node-backup.service"): + raise DeliveryError("fixed backup operator unit required") + if { + k: (group / k).read_text().strip() + for k in ("memory.max", "memory.swap.max", "pids.max", "cpu.max") + } != { + "memory.max": "268435456", + "memory.swap.max": "0", + "pids.max": "32", + "cpu.max": "50000 100000", + }: + raise DeliveryError("actual backup operator limits differ") + node = Path("/sys/fs/cgroup/kfonlinejudge.slice") + if { + k: (node / k).read_text().strip() + for k in ("memory.max", "memory.swap.max", "pids.max", "cpu.max") + } != { + "memory.max": "4026531840", + "memory.swap.max": "0", + "pids.max": "384", + "cpu.max": "200000 100000", + }: + raise DeliveryError("actual backup node limits differ") + for key, bound in ( + (resource.RLIMIT_CORE, 0), + (resource.RLIMIT_FSIZE, backup_manifest.MAX_FILE_BYTES), + (resource.RLIMIT_NOFILE, 256), + (resource.RLIMIT_STACK, 16 * 1024**2), + ): + if resource.getrlimit(key) != (bound, bound): + raise DeliveryError("backup process limit differs") + + +def capture(backup_id: str, references_file: Path, references_sha: str) -> dict: + guarded() + selected = install.selection() + payload = install.RELEASES / selected["release_id"] / "payload" + for name in ("backup.py", "backup_manifest.py"): + if digest(payload / "delivery/scripts/release" / name) != digest( + Path(__file__).parent / name + ): + raise DeliveryError("backup operator differs from selected release") + configuration.private(references_file) + if digest(references_file) != sha256(references_sha): + raise DeliveryError("separate private artifact receipt differs") + refs = backup_manifest.validate_references(backup_manifest.read_metadata(references_file)) + cut = read_json(lifecycle.LAST, 32768) + stopped(selected, cut) + artifact_root = SOURCES["artifacts"].stat() + if refs["boot_id"] != cut["boot_id"] or refs["storage_identity"] != { + "device": artifact_root.st_dev, + "inode": artifact_root.st_ino, + "uid": artifact_root.st_uid, + }: + raise DeliveryError("artifact inventory was not observed on this source storage/boot") + volume() + lock = lifecycle.STATE / "maintenance.lock" + configuration.private(lock) + descriptor = os.open(lock, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) + deadline = time.monotonic() + 1700 + target = ROOT / identifier(backup_id) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + stopped(selected, cut) + if target.exists() or target.is_symlink(): + raise DeliveryError("backup target already exists; incomplete backups retained") + target.mkdir(mode=0o700) + tree = target / "payload" + tree.mkdir(mode=0o700) + dirs, files, total = [], [], 0 + for domain, source_root in SOURCES.items(): + if source_root.is_symlink() or not source_root.is_dir(): + raise DeliveryError("a required backup source domain is absent or linked") + for directory, names, members in os.walk(source_root, followlinks=False): + if time.monotonic() >= deadline: + raise DeliveryError("private backup capture deadline") + source_dir = Path(directory) + relative = backup_manifest.name( + domain + + ( + "/" + source_dir.relative_to(source_root).as_posix() + if source_dir != source_root + else "" + ) + ) + info = source_dir.lstat() + entry = { + "path": relative, + "uid": info.st_uid, + "gid": info.st_gid, + "mode": stat.S_IMODE(info.st_mode), + } + backup_manifest._member(entry, set(), directory=True) + if not stat.S_ISDIR(info.st_mode): + raise DeliveryError("backup source directory changed type") + destination = tree / relative + destination.mkdir(mode=0o700) + dirs.append(entry) + if len(dirs) + len(files) > backup_manifest.MAX_ENTRIES: + raise DeliveryError("backup directory inventory exceeds bound") + for member in names: + if (source_dir / member).is_symlink(): + raise DeliveryError("linked backup directory refused") + for member in sorted(members): + p = source_dir / member + before = p.lstat() + if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1: + raise DeliveryError("special or linked backup source refused") + entry = { + "path": backup_manifest.name(relative + "/" + member), + "uid": before.st_uid, + "gid": before.st_gid, + "mode": stat.S_IMODE(before.st_mode), + "bytes": before.st_size, + } + backup_manifest._member(entry, set(), directory=False) + if before.st_size > backup_manifest.MAX_FILE_BYTES: + raise DeliveryError("backup individual file exceeds bound") + total += before.st_size + if ( + total > backup_manifest.MAX_BYTES + or len(dirs) + len(files) >= backup_manifest.MAX_ENTRIES + or os.statvfs(ROOT).f_frsize * os.statvfs(ROOT).f_bavail + < before.st_size + 64 * 1024**2 + ): + raise DeliveryError("private backup capacity exhausted") + fd = os.open(p, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC) + h = hashlib.sha256() + output = os.open( + destination / member, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, + ) + with ( + os.fdopen(fd, "rb") as incoming, + os.fdopen(output, "wb") as outgoing, + ): + observed = os.fstat(incoming.fileno()) + if any( + getattr(before, k) != getattr(observed, k) + for k in ( + "st_dev", + "st_ino", + "st_size", + "st_mtime_ns", + "st_ctime_ns", + ) + ): + raise DeliveryError("backup source changed before capture") + count = 0 + while block := incoming.read(65536): + count += len(block) + if count > before.st_size or time.monotonic() >= deadline: + raise DeliveryError("backup source or deadline changed") + h.update(block) + outgoing.write(block) + after = os.fstat(incoming.fileno()) + if count != before.st_size or any( + getattr(before, k) != getattr(after, k) + for k in ( + "st_dev", + "st_ino", + "st_size", + "st_mtime_ns", + "st_ctime_ns", + ) + ): + raise DeliveryError("backup source changed during capture") + outgoing.flush() + os.fsync(outgoing.fileno()) + os.fchmod(outgoing.fileno(), 0o600) + entry["sha256"] = h.hexdigest() + files.append(entry) + install.sync_directory(destination) + stopped(selected, cut) + if install.selection() != selected or read_json(lifecycle.LAST, 32768) != cut: + raise DeliveryError("selected cut changed during backup") + manifest = read_json(payload.parent / "manifest.json") + now = int(time.time()) + value = backup_manifest.validate( + { + "schema_version": 1, + "format": backup_manifest.FORMAT, + "backup_id": backup_id, + "source_sha": manifest["source_sha"], + "manifest_sha256": selected["manifest_sha256"], + "configuration_id": selected["configuration_id"], + "created_unix_seconds": now, + "expires_unix_seconds": now + 7 * 86400, + "consistent_cut": cut, + "selected": selected, + "artifact_references": refs, + "images": manifest["images"], + "directories": sorted(dirs, key=lambda x: x["path"]), + "files": sorted(files, key=lambda x: x["path"]), + "payload_bytes": total, + "restored": False, + "execution_verified": False, + } + ) + raw = canonical(value) + if len(raw) > backup_manifest.MAX_METADATA_BYTES: + raise DeliveryError("private backup manifest exceeds bound") + install.exclusive_file(target / "manifest.json", raw, 0o600) + install.sync_directory(tree) + install.sync_directory(target) + install.sync_directory(ROOT) + fingerprint = digest(target / "manifest.json") + backup_manifest.verify(target, fingerprint) + return { + "backup_id": backup_id, + "private_manifest_sha256": fingerprint, + "payload_bytes": total, + "source_sha": manifest["source_sha"], + "captured": True, + "restored": False, + "execution_verified": False, + } + finally: + os.close(descriptor) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Private bounded cold backup operator") + sub = parser.add_subparsers(dest="action", required=True) + create = sub.add_parser("capture") + create.add_argument("--backup-id", required=True) + create.add_argument("--references", type=Path, required=True) + create.add_argument("--references-sha256", required=True) + create.add_argument("--inside", action="store_true") + check = sub.add_parser("verify") + check.add_argument("--backup-id", required=True) + check.add_argument("--manifest-sha256", required=True) + args = parser.parse_args() + try: + if os.getuid() != 0 or os.geteuid() != 0: + raise DeliveryError("explicit root backup operator required") + if args.action == "verify": + value = backup_manifest.verify(ROOT / identifier(args.backup_id), args.manifest_sha256) + result = { + "verified": True, + "backup_id": value["backup_id"], + "restored": False, + "execution_verified": False, + } + elif args.inside: + result = capture(args.backup_id, args.references, args.references_sha256) + else: + # Caller provides only an ID/private digest; no program, identity, + # source-domain or resource override is accepted by this launcher. + identifier(args.backup_id) + configuration.private(args.references) + if digest(args.references) != sha256(args.references_sha256): + raise DeliveryError("private artifact inventory receipt differs") + selected = install.selection() + payload = install.RELEASES / selected["release_id"] / "payload" + current = manager( + "show", + "kf-node-backup.service", + "--property=LoadState,ActiveState,MainPID", + ) + if current != { + "LoadState": "not-found", + "ActiveState": "inactive", + "MainPID": "0", + }: + raise DeliveryError("unknown or active backup operator") + command = [ + "/usr/bin/systemd-run", + "--quiet", + "--wait", + "--pipe", + "--collect", + "--unit=kf-node-backup.service", + "--slice=kfonlinejudge.slice", + ] + for key, value in { + "User": "root", + "Group": "root", + "MemoryMax": "256M", + "MemorySwapMax": "0", + "TasksMax": "32", + "CPUQuota": "50%", + "RuntimeMaxSec": "1800", + "TimeoutStopSec": "15", + "KillMode": "control-group", + "Restart": "no", + "LimitCORE": "0", + "LimitFSIZE": "256M", + "LimitNOFILE": "256", + "LimitSTACK": "16M", + "UMask": "0077", + "NoNewPrivileges": "yes", + "CapabilityBoundingSet": "CAP_DAC_READ_SEARCH", + "AmbientCapabilities": "CAP_DAC_READ_SEARCH", + "ProtectSystem": "strict", + "ProtectHome": "yes", + "PrivateTmp": "yes", + "PrivateDevices": "yes", + "ProtectControlGroups": "yes", + "RestrictAddressFamilies": "AF_UNIX", + "ReadWritePaths": str(ROOT), + }.items(): + command += ["--property=" + key + "=" + value] + command += [ + "/usr/bin/env", + "-i", + "PATH=/usr/bin:/bin", + "LC_ALL=C.UTF-8", + *python_environment(payload).split(), + str(payload / "python/bin/python3.13"), + "-s", + "-P", + "-m", + "scripts.release.backup", + "capture", + "--inside", + "--backup-id", + args.backup_id, + "--references", + str(args.references), + "--references-sha256", + args.references_sha256, + ] + result = subprocess.run(command, capture_output=True, timeout=1820, check=False) + if result.returncode or len(result.stdout) > 4096: + raise DeliveryError( + "cold backup capture failed; source remains stopped and partial target retained" + ) + result = json.loads(result.stdout) + print(json.dumps(result, sort_keys=True)) + except (OSError, ValueError, subprocess.SubprocessError) as error: + parser.exit( + 1, + "Private backup refused: " + + (str(error) if isinstance(error, DeliveryError) else type(error).__name__) + + "\n", + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/backup_manifest.py b/scripts/release/backup_manifest.py new file mode 100644 index 0000000..19e29e2 --- /dev/null +++ b/scripts/release/backup_manifest.py @@ -0,0 +1,341 @@ +"""Private cold-backup directory format; never a public release or CI artifact.""" + +import json +import os +import re +import stat +from pathlib import Path + +from .policy import ( + DeliveryError, + digest, + fields, + integer, + invalid_constant, + sha256, + unique_object, +) + +FORMAT = "kf.private.cold-backup.v1" +MAX_BYTES = 7 * 1024**3 +MAX_FILE_BYTES = 256 * 1024**2 +MAX_ENTRIES = 50000 +MAX_METADATA_BYTES = 16 * 1024**2 +DOMAINS = { + "postgres", + "artifacts", + "imports", + "app-config", + "judge-config", + "factory", + "operations", + "releases", + "images", + "dependencies", + "proxy", +} + + +def read_metadata(path: Path) -> dict: + """Private inventories can exceed the public bundle's structure count. + + Byte, depth and entry bounds remain explicit before any restore/capture use. + """ + with path.open("rb") as stream: + raw = stream.read(MAX_METADATA_BYTES + 1) + if len(raw) > MAX_METADATA_BYTES: + raise DeliveryError("private backup metadata byte limit") + try: + value = json.loads(raw, object_pairs_hook=unique_object, parse_constant=invalid_constant) + pending = [(value, 0)] + count = 0 + while pending: + item, depth = pending.pop() + count += 1 + if depth > 32 or count > 1_000_000: + raise DeliveryError("private backup metadata structure limit") + if isinstance(item, dict): + pending.extend((member, depth + 1) for member in item.values()) + elif isinstance(item, list): + pending.extend((member, depth + 1) for member in item) + return value + except (UnicodeError, json.JSONDecodeError, RecursionError) as error: + raise DeliveryError("invalid private backup metadata") from error + + +def name(value: str) -> str: + if not isinstance(value, str) or not 1 <= len(value) <= 512: + raise DeliveryError("invalid private backup member") + parts = value.split("/") + if parts[0] not in DOMAINS or any( + p in {"", ".", "..", ".git", ".ssh", ".localplan", "docs"} + or re.fullmatch(r"[A-Za-z0-9_.+@-]{1,255}", p) is None + for p in parts + ): + raise DeliveryError("private backup member outside fixed domains") + return value + + +def validate_references(value: dict) -> dict: + fields( + value, + { + "schema_version", + "kind", + "read_only", + "quiescence_verified", + "boot_id", + "storage_identity", + "objects", + }, + "private artifact references", + ) + if ( + type(value["schema_version"]) is not int + or value["schema_version"] != 1 + or value["kind"] != "kf.private.artifact-backup-inventory" + or value["read_only"] is not True + or value["quiescence_verified"] is not False + or not isinstance(value["objects"], list) + or len(value["objects"]) > 65536 + ): + raise DeliveryError("artifact backup inventory authority differs") + from uuid import UUID + + if ( + not isinstance(value["boot_id"], str) + or str(UUID(value["boot_id"])) != value["boot_id"] + or UUID(value["boot_id"]).int == 0 + ): + raise DeliveryError("artifact backup boot identity differs") + identity = fields( + value["storage_identity"], + {"device", "inode", "uid"}, + "artifact storage identity", + ) + integer(identity["device"], 1, 2**64 - 1, "artifact storage device") + integer(identity["inode"], 1, 2**64 - 1, "artifact storage inode") + if type(identity["uid"]) is not int or identity["uid"] != 1200: + raise DeliveryError("artifact storage identity is not the platform owner") + seen = set() + for item in value["objects"]: + fields(item, {"digest", "bytes"}, "private artifact reference") + sha256(item["digest"]) + integer(item["bytes"], 0, 64 * 1024**2, "artifact reference bytes") + if item["digest"] in seen: + raise DeliveryError("duplicate backup artifact reference") + seen.add(item["digest"]) + return value + + +def validate(value: dict) -> dict: + fields( + value, + { + "schema_version", + "format", + "backup_id", + "source_sha", + "manifest_sha256", + "configuration_id", + "created_unix_seconds", + "expires_unix_seconds", + "consistent_cut", + "selected", + "artifact_references", + "images", + "directories", + "files", + "payload_bytes", + "restored", + "execution_verified", + }, + "private backup", + ) + if ( + type(value["schema_version"]) is not int + or value["schema_version"] != 1 + or value["format"] != FORMAT + or not isinstance(value["backup_id"], str) + or re.fullmatch(r"bk-[0-9a-f]{32}", value["backup_id"]) is None + or not isinstance(value["source_sha"], str) + or re.fullmatch(r"[0-9a-f]{40}", value["source_sha"]) is None + or not isinstance(value["configuration_id"], str) + or re.fullmatch(r"cfg-[0-9a-f]{24}", value["configuration_id"]) is None + or value["restored"] is not False + or value["execution_verified"] is not False + ): + raise DeliveryError("private backup identity or authority differs") + sha256(value["manifest_sha256"]) + start = integer(value["created_unix_seconds"], 1, 2**63 - 1, "backup time") + end = integer(value["expires_unix_seconds"], start + 1, start + 7 * 86400, "backup retention") + if end - start != 7 * 86400: + raise DeliveryError("fixed seven-day private backup retention required") + from .lifecycle import projection_empty, validate_record + + cut = validate_record(value["consistent_cut"]) + if ( + cut["source_sha"] != value["source_sha"] + or cut["configuration_id"] != value["configuration_id"] + or cut["action"] not in {"drain", "stop"} + or cut["phase"] != "complete" + or cut["success"] is not True + or cut["root_cleanup_confirmed"] is not True + or "drain_projection" not in cut + or not projection_empty(cut["drain_projection"]) + ): + raise DeliveryError("cold backup lacks a confirmed owner-derived cut") + selected = fields( + value["selected"], + { + "schema_version", + "release_id", + "manifest_sha256", + "configuration_id", + "receipt_sha256", + }, + "backup selected binding", + ) + if ( + type(selected["schema_version"]) is not int + or selected["schema_version"] != 1 + or selected["release_id"] != "sd-" + value["source_sha"][:12] + or selected["manifest_sha256"] != value["manifest_sha256"] + or selected["configuration_id"] != value["configuration_id"] + ): + raise DeliveryError("backup selected binding differs") + sha256(selected["receipt_sha256"]) + validate_references(value["artifact_references"]) + if value["artifact_references"]["boot_id"] != cut["boot_id"]: + raise DeliveryError("artifact snapshot and cold cut are from different boots") + fields(value["images"], {"gnu", "python313", "java21"}, "backup image identities") + for identity in value["images"].values(): + fields( + identity, + { + "toolchain_id", + "image_kind", + "image_revision", + "runtime_version", + "image_digest", + }, + "backup runtime identity", + ) + sha256(identity["image_digest"]) + integer(identity["image_revision"], 1, 2**32 - 1, "backup image revision") + for key in ("toolchain_id", "image_kind", "runtime_version"): + if ( + not isinstance(identity[key], str) + or re.fullmatch(r"[A-Za-z0-9_.+-]{1,128}", identity[key]) is None + ): + raise DeliveryError("backup runtime label differs") + dirs, files = value["directories"], value["files"] + if ( + not isinstance(dirs, list) + or not isinstance(files, list) + or not 1 <= len(dirs) + len(files) <= MAX_ENTRIES + ): + raise DeliveryError("backup inventory count exceeds bound") + seen = set() + for item in dirs: + fields(item, {"path", "uid", "gid", "mode"}, "backup directory") + _member(item, seen, directory=True) + total = 0 + for item in files: + fields(item, {"path", "uid", "gid", "mode", "bytes", "sha256"}, "backup file") + _member(item, seen, directory=False) + total += integer(item["bytes"], 0, MAX_FILE_BYTES, "backup file bytes") + sha256(item["sha256"]) + if ( + type(value["payload_bytes"]) is not int + or value["payload_bytes"] != total + or total > MAX_BYTES + ): + raise DeliveryError("private backup aggregate bytes differ") + directories = {item["path"] for item in dirs} + if not DOMAINS <= directories: + raise DeliveryError("private backup omitted a required source domain") + for path in seen: + for parent in Path(path).parents: + if parent.as_posix() != "." and parent.as_posix() not in directories: + raise DeliveryError("private backup parent missing") + lookup = {item["path"]: item for item in files} + for reference in value["artifact_references"]["objects"]: + item = lookup.get("artifacts/" + reference["digest"]) + if ( + item is None + or item["bytes"] != reference["bytes"] + or item["sha256"] != reference["digest"] + ): + raise DeliveryError("registered source or testcase absent from backup") + if "postgres/PG_VERSION" not in lookup or "postgres/global/pg_control" not in lookup: + raise DeliveryError("private backup lacks a cold PostgreSQL cluster") + return value + + +def _member(item: dict, seen: set[str], *, directory: bool) -> None: + path = name(item["path"]) + if path in seen: + raise DeliveryError("duplicate private backup member") + seen.add(path) + integer(item["uid"], 0, 1205, "backup member uid") + integer(item["gid"], 0, 1205, "backup member gid") + if item["uid"] not in {0, 1200, 1201, 1202, 1203, 1204, 1205} or ( + item["gid"] not in ({0, 1203, 1204, 1205} if item["uid"] == 0 else {item["uid"]}) + ): + raise DeliveryError("unsupported backup OS identity") + mode = item["mode"] + if type(mode) is not int or mode not in ( + {0o555, 0o700, 0o711, 0o755} + if directory + else {0o400, 0o444, 0o600, 0o640, 0o644, 0o555, 0o755} + ): + raise DeliveryError("backup member privilege bits differ") + + +def verify(root: Path, expected: str) -> dict: + """Expected digest is a separate private operator receipt, never self-trust.""" + from . import configuration, preflight + + preflight.protected(root, directory=True) + if stat.S_IMODE(root.stat().st_mode) != 0o700: + raise DeliveryError("private backup directory required") + path = root / "manifest.json" + configuration.private(path) + if digest(path) != sha256(expected) or {p.name for p in root.iterdir()} != { + "manifest.json", + "payload", + }: + raise DeliveryError("private backup receipt or layout differs") + value = validate(read_metadata(path)) + found_files, found_dirs = set(), set() + payload = root / "payload" + preflight.protected(payload, directory=True) + for directory, names, files in os.walk(payload, followlinks=False): + for member in names + files: + p = Path(directory) / member + info = p.lstat() + relative = name(p.relative_to(payload).as_posix()) + if info.st_uid != 0 or info.st_gid != 0 or p.is_symlink(): + raise DeliveryError("private backup member ownership differs") + if stat.S_ISDIR(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o700: + found_dirs.add(relative) + elif ( + stat.S_ISREG(info.st_mode) + and info.st_nlink == 1 + and stat.S_IMODE(info.st_mode) == 0o600 + ): + found_files.add(relative) + else: + raise DeliveryError("private backup special or non-private member") + if len(found_files) + len(found_dirs) > MAX_ENTRIES: + raise DeliveryError("private backup traversal exceeds bound") + if found_dirs != {x["path"] for x in value["directories"]} or found_files != { + x["path"] for x in value["files"] + }: + raise DeliveryError("private backup inventory differs") + for item in value["files"]: + p = payload / item["path"] + if p.stat().st_size != item["bytes"] or digest(p) != item["sha256"]: + raise DeliveryError("private backup byte verification failed") + return value