"""Root dispatch/receipt for an ordinary, finite and isolated DB identity recovery.""" import argparse import fcntl import json import os import re import stat import subprocess from contextlib import contextmanager from pathlib import Path from urllib.parse import urlsplit from . import ( backup, backup_manifest, configuration, dependency_initialize, install, manifest, preflight, recovery_identity, restore, restore_policy, ) from .policy import DeliveryError, canonical, digest, fields, integer, read_json, sha256 from .services import python_environment def validate_complete( prepared: dict, data: dict, identity: dict, recovery_id: str, data_sha: str ) -> None: """A completed historical recovery survives reboot; an incomplete one never does.""" fields( prepared, { "schema_version", "recovery_id", "backup_id", "private_backup_sha256", "source_sha", "configuration_id", "boot_id", "data_restored", "database_credentials_rotated", "services_started", "execution_verified", }, "prepared recovery", ) if ( type(prepared["schema_version"]) is not int or prepared["schema_version"] != 1 or prepared["recovery_id"] != restore.identifier(recovery_id) or any( prepared[key] is not False for key in ( "data_restored", "database_credentials_rotated", "services_started", "execution_verified", ) ) or not isinstance(prepared["source_sha"], str) or re.fullmatch(r"[0-9a-f]{40}", prepared["source_sha"]) is None or not isinstance(prepared["configuration_id"], str) or re.fullmatch(r"cfg-[0-9a-f]{24}", prepared["configuration_id"]) is None or not isinstance(prepared["boot_id"], str) or re.fullmatch( r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", prepared["boot_id"], ) is None ): raise DeliveryError("incomplete or foreign prepared recovery; admission refused") validate_data_receipt(data, recovery_id, prepared, prepared["source_sha"], prepared["boot_id"]) if any(data[key] != value for key, value in prepared.items() if key != "data_restored"): raise DeliveryError("recovery data receipt differs from prepared binding") fields( identity, { "schema_version", "recovery_id", "source_sha", "configuration_id", "boot_id", "data_receipt_sha256", "bootstrap_sha256", "cluster_identity", "temporary_server_stopped", "database_credentials_rotated", "ordinary_identity_readback", "services_started", "schema_migrated", "execution_state_changed", "execution_verified", }, "completed recovery identity", ) if ( any( identity[key] != prepared[key] for key in ( "schema_version", "recovery_id", "source_sha", "configuration_id", "boot_id", ) ) or type(identity["schema_version"]) is not int or identity["data_receipt_sha256"] != sha256(data_sha) or identity["temporary_server_stopped"] is not True or identity["database_credentials_rotated"] is not True or any( identity[key] is not False for key in ( "services_started", "schema_migrated", "execution_state_changed", "execution_verified", ) ) ): raise DeliveryError("recovery identity is incomplete or unbound; admission refused") sha256(identity["bootstrap_sha256"]) dependency_initialize.validate_cluster_identity(identity["cluster_identity"]) read_result(canonical(identity["ordinary_identity_readback"])) @contextmanager def admission_guard(): """Hold a read-only shared lock through startup, refusing all partial recoveries.""" preflight.protected(restore.STATE, directory=True) if stat.S_IMODE(restore.STATE.stat().st_mode) != 0o700: raise DeliveryError("private recovery state required before admission") lock = restore.STATE / "restore.lock" preflight.protected(lock) configuration.private(lock) if lock.stat().st_size != 0: raise DeliveryError("unknown recovery lock contents") descriptor = os.open(lock, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) try: fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB) for count, directory in enumerate(restore.STATE.iterdir(), start=1): if count > 9: raise DeliveryError("recovery history exceeds startup observation bound") if directory == lock: continue restore.identifier(directory.name) preflight.protected(directory, directory=True) if stat.S_IMODE(directory.stat().st_mode) != 0o700: raise DeliveryError("private recovery record directory required") records = {} for name in ("prepared.json", "data-restored.json", "identity-recovered.json"): path = directory / name preflight.protected(path) configuration.private(path) records[name] = read_json(path, 16384) validate_complete( records["prepared.json"], records["data-restored.json"], records["identity-recovered.json"], directory.name, digest(directory / "data-restored.json"), ) yield finally: os.close(descriptor) def bootstrap_fingerprint() -> str: """Read the current cluster's proven credential marker, including restored nodes.""" candidates = [] cluster = dependency_initialize.cluster_identity() with admission_guard(): initial = dependency_initialize.RECEIPT if initial.exists() or initial.is_symlink(): preflight.protected(initial) configuration.private(initial) candidates.append(dependency_initialize.validate_receipt(read_json(initial, 8192))) for directory in restore.STATE.iterdir(): if directory.name == "restore.lock": continue # admission_guard already validated all linked completion receipts. candidates.append(read_json(directory / "identity-recovered.json", 16384)) if len(candidates) != 1 or candidates[0]["cluster_identity"] != cluster: raise DeliveryError("current DB bootstrap fingerprint is absent or ambiguous") return sha256(candidates[0]["bootstrap_sha256"]) def validate_data_receipt( value: dict, recovery_id: str, selected: dict, source_sha: str, boot: str ) -> dict: fields( value, { "schema_version", "recovery_id", "backup_id", "private_backup_sha256", "source_sha", "configuration_id", "boot_id", "data_restored", "database_credentials_rotated", "services_started", "execution_verified", "restored_payload_bytes", "restored_files", "restored_directories", "historical_configuration_active", "factory_replay_history_retained", }, "cold data restoration receipt", ) if ( type(value["schema_version"]) is not int or value["schema_version"] != 1 or value["recovery_id"] != restore.identifier(recovery_id) or value["source_sha"] != source_sha or value["configuration_id"] != selected["configuration_id"] or value["boot_id"] != boot or value["data_restored"] is not True or value["database_credentials_rotated"] is not False or value["services_started"] is not False or value["execution_verified"] is not False or value["historical_configuration_active"] is not False or value["factory_replay_history_retained"] is not True ): raise DeliveryError("cold data restoration is not this boot's selected complete data step") backup.identifier(value["backup_id"]) sha256(value["private_backup_sha256"]) integer(value["restored_payload_bytes"], 1, 7 * 1024**3, "restored bytes") for key in ("restored_files", "restored_directories"): integer(value[key], 1, 50000, "restored entries") if value["restored_files"] + value["restored_directories"] > 50000: raise DeliveryError("restored inventory limit differs") return value def recover(recovery_id: str, data_sha: str, password_file: Path, password_sha: str) -> dict: # Share the data step's lock with admission. A partial password rotation # must not race service startup or another restoration operation. lock = restore.STATE / "restore.lock" configuration.private(lock) if lock.stat().st_size != 0: raise DeliveryError("unknown recovery lock contents") descriptor = os.open(lock, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) try: fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) return recover_locked(recovery_id, data_sha, password_file, password_sha) finally: os.close(descriptor) def recover_locked(recovery_id: str, data_sha: str, password_file: Path, password_sha: str) -> dict: if (os.getuid(), os.geteuid(), os.getgid()) != (0, 0, 0): raise DeliveryError("explicit root recovery identity dispatcher required") restore.identifier(recovery_id) restore.stopped() selected = install.selection() payload = install.RELEASES / selected["release_id"] / "payload" data = manifest.verify(payload.parent, selected["manifest_sha256"]) for name in ( "recovery_database.py", "recovery_identity.py", "restore.py", "restore_policy.py", "dependencies.py", "dependency_policy.py", "layout.py", ): if digest(payload / "delivery/scripts/release" / name) != digest( Path(__file__).parent / name ): raise DeliveryError("recovery identity program differs from selected delivery") directory = restore.STATE / recovery_id preflight.protected(directory, directory=True) preflight_path = directory / "data-restored.json" preflight.protected(preflight_path) configuration.private(preflight_path) if digest(preflight_path) != sha256(data_sha): raise DeliveryError("separate cold data restoration fingerprint differs") boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip() recorded = validate_data_receipt( read_json(preflight_path, 8192), recovery_id, selected, data["source_sha"], boot ) preflight.protected(password_file) configuration.private(password_file) if re.fullmatch(r"/[A-Za-z0-9_./-]+", str(password_file)) is None or digest( password_file ) != sha256(password_sha): raise DeliveryError("private fresh DB bootstrap fingerprint/path differs") dependency_initialize.validate_password(password_file.read_bytes()) # Bind the latest cluster fingerprint through the separately hashed private # snapshot. Restored nodes need not have an initial-cluster receipt. snapshot_path = backup.ROOT / recorded["backup_id"] / "manifest.json" preflight.protected(snapshot_path) configuration.private(snapshot_path) if digest(snapshot_path) != recorded["private_backup_sha256"]: raise DeliveryError("private recovery snapshot fingerprint changed") previous = backup_manifest.validate(backup_manifest.read_metadata(snapshot_path)) if previous["backup_id"] != recorded["backup_id"]: raise DeliveryError("private recovery snapshot ID differs") if previous["database_bootstrap_sha256"] == password_sha: raise DeliveryError("new recovery DB bootstrap secret required") current = restore_policy.snapshot( { name: configuration.installed_path( selected["release_id"], selected["configuration_id"], name ) for name in configuration.CONFIG_FILES } ) fresh_bootstrap = password_file.read_text().strip() if any( fresh_bootstrap == urlsplit(endpoint).password for kind in ("database", "amqp") for endpoint in current["endpoints"][kind].values() ): raise DeliveryError("recovery bootstrap must be distinct from all service credentials") credential = directory / "identity-connections.json" install.exclusive_file(credential, canonical(current["endpoints"]), 0o600) if (directory / "identity-recovered.json").exists(): raise DeliveryError("DB identities already recovered; no implicit repeat/reset") unit = recovery_identity.UNIT status = backup.manager("show", unit, "--property=LoadState,ActiveState,MainPID") if status != {"LoadState": "not-found", "ActiveState": "inactive", "MainPID": "0"}: raise DeliveryError("unknown or active DB identity recovery unit") if recovery_identity.RUNTIME.exists() or recovery_identity.RUNTIME.is_symlink(): raise DeliveryError("unknown residual DB identity recovery runtime directory") identity = dependency_initialize.cluster_identity() description = "KF fixed DB recovery " + recovery_id + "." + selected["configuration_id"] command = [ "/usr/bin/systemd-run", "--quiet", "--wait", "--pipe", "--collect", "--unit=" + unit, "--description=" + description, "--slice=kfonlinejudge.slice", ] for key, setting in { "Type": "exec", "User": "postgres", "Group": "postgres", "MemoryMax": "512M", "MemorySwapMax": "0", "CPUQuota": "100%", "TasksMax": "64", "RuntimeMaxSec": "240", "TimeoutStopSec": "45", "KillMode": "control-group", "KillSignal": "SIGINT", "Restart": "no", "LimitCORE": "0", "LimitFSIZE": "256M", "LimitNOFILE": "256", "LimitSTACK": "16M", "NoNewPrivileges": "yes", "CapabilityBoundingSet": "", "AmbientCapabilities": "", "UMask": "0077", "PrivateTmp": "yes", "PrivateDevices": "yes", "PrivateNetwork": "yes", "PrivatePIDs": "yes", "ProtectHome": "yes", "ProtectSystem": "strict", "ProtectControlGroups": "yes", "RestrictAddressFamilies": "AF_UNIX", "RuntimeDirectory": recovery_identity.RUNTIME.name, "RuntimeDirectoryMode": "0700", "ReadWritePaths": str(dependency_initialize.DATA.parent), }.items(): command.append("--property=" + key + "=" + setting) command += [ "--property=LoadCredential=connections:" + str(credential), "--property=LoadCredential=bootstrap:" + str(password_file), "/usr/bin/env", "-i", "PATH=/usr/bin:/bin", "LC_ALL=C.UTF-8", *python_environment(payload).split(), str(payload / "python/bin/python3.13"), "-s", "-P", "-m", "scripts.release.recovery_identity", ] result = None dispatched = False try: dispatched = True completed = subprocess.run( command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=290, check=False ) if completed.returncode or len(completed.stdout) > 16384: raise DeliveryError("DB identity recovery failed; target remains stopped") result = read_result(completed.stdout) finally: if dispatched: active = backup.manager( "show", unit, "--property=LoadState,ActiveState,MainPID,Description,Transient" ) if active.get("LoadState") != "not-found": if active.get("Description") != description or active.get("Transient") != "yes": raise DeliveryError("foreign DB recovery unit; refused cleanup") from . import service_install service_install.systemctl("stop", unit) restore.stopped() if ( recovery_identity.RUNTIME.exists() or dependency_initialize.DATA.joinpath("postmaster.pid").exists() ): raise DeliveryError("DB identity recovery shutdown/runtime cleanup unconfirmed") if install.selection() != selected or dependency_initialize.cluster_identity() != identity: raise DeliveryError("recovery configuration or cluster identity changed") if digest(preflight_path) != data_sha: raise DeliveryError("cold data restoration receipt changed") record = { "schema_version": 1, "recovery_id": recovery_id, "source_sha": data["source_sha"], "configuration_id": selected["configuration_id"], "boot_id": boot, "data_receipt_sha256": data_sha, "bootstrap_sha256": password_sha, "cluster_identity": identity, "temporary_server_stopped": True, "database_credentials_rotated": True, "ordinary_identity_readback": result, "services_started": False, "schema_migrated": False, "execution_state_changed": False, "execution_verified": False, } install.exclusive_file(directory / "identity-recovered.json", canonical(record), 0o600) return { "recovery_id": recovery_id, "database_credentials_rotated": True, "receipt_sha256": digest(directory / "identity-recovered.json"), "temporary_server_stopped": True, "schema_migrated": False, "services_started": False, "execution_verified": False, } def read_result(raw: bytes) -> dict: from .operator_units import decode value = decode(raw, 16384) fields( value, { "dependency", "actual_cgroup_limits", "data_filesystem_bytes", "execution_verified", "rotated_roles", "role_authority_sha256", "authenticated", "schema_migrated", "execution_state_changed", "effective_settings", "temporary_server_stopped", "services_started", }, "ordinary DB recovery result", ) if ( value["dependency"] != "postgres" or value["rotated_roles"] != sorted(["postgres", *configuration.PROFILES.values()]) or value["actual_cgroup_limits"] != { "memory.max": "536870912", "memory.swap.max": "0", "pids.max": "64", "cpu.max": "100000 100000", } or value["authenticated"] is not True or value["temporary_server_stopped"] is not True or value["schema_migrated"] is not False or value["execution_state_changed"] is not False or value["services_started"] is not False or value["execution_verified"] is not False ): raise DeliveryError("ordinary DB recovery result is incomplete or unproven") integer( value["data_filesystem_bytes"], 2 * 1024**3 * 9 // 10, 2 * 1024**3, "DB recovery volume" ) sha256(value["role_authority_sha256"]) fields(value["effective_settings"], set(recovery_identity.recovery_settings()), "PG settings") for item in value["effective_settings"].values(): fields(item, {"setting", "unit"}, "PG effective setting") if ( not isinstance(item["setting"], str) or item["unit"] is not None and not isinstance(item["unit"], str) ): raise DeliveryError("invalid effective PostgreSQL setting") recovery_identity.check_settings( [{"name": key, **item} for key, item in value["effective_settings"].items()] ) return value def main() -> None: parser = argparse.ArgumentParser( description="Explicit ordinary DB identity recovery after cold restoration" ) parser.add_argument("--recovery-id", required=True) parser.add_argument("--data-receipt-sha256", required=True) parser.add_argument("--password-file", type=Path, required=True) parser.add_argument("--password-sha256", required=True) args = parser.parse_args() try: value = recover( args.recovery_id, args.data_receipt_sha256, args.password_file, args.password_sha256 ) print(json.dumps(value, sort_keys=True)) except (OSError, ValueError, subprocess.SubprocessError) as error: parser.exit( 1, "DB recovery refused: " + (str(error) if isinstance(error, DeliveryError) else type(error).__name__) + "\n", ) if __name__ == "__main__": main()