diff --git a/backend/tests/test_delivery_backup.py b/backend/tests/test_delivery_backup.py index bebcd0a..6d20aeb 100644 --- a/backend/tests/test_delivery_backup.py +++ b/backend/tests/test_delivery_backup.py @@ -302,3 +302,48 @@ def test_delayed_filesystem_write_error_is_not_success(tmp_path, monkeypatch): monkeypatch.setattr(backup_objects.ctypes, "CDLL", lambda *_, **__: Library()) with pytest.raises(policy.DeliveryError, match="filesystem synchronization failed"): backup_objects.sync_filesystem(tmp_path, time.monotonic() + 10) + + +@pytest.mark.parametrize("missing", ["releases/older", "releases/older/nested"]) +def test_every_directory_ancestor_is_required_even_when_file_parent_exists(snapshot, missing): + value = object_format(snapshot) + value["directories"] = value["directories"] + [ + {"path": path, "uid": 0, "gid": 0, "mode": 0o700} + for path in ("releases/older", "releases/older/nested") + if path != missing + ] + value["files"] = [ + {**row, "path": "releases/older/nested/file"} if row["path"] == "releases/first" else row + for row in value["files"] + ] + with pytest.raises(policy.DeliveryError, match="private backup parent missing"): + backup_manifest.validate(value) + + +def test_v4_complete_version_inventory_keeps_an_explicit_logical_entry_bound(snapshot): + value = object_format(snapshot) + count = backup_manifest.MAX_LOGICAL_ENTRIES - len(value["directories"]) + template = {**value["files"][0], "bytes": 1, "sha256": hashlib.sha256(b"x").hexdigest()} + value["files"] = [{**template, "path": row["path"]} for row in value["files"][:2]] + [ + {**template, "path": "releases/item-" + str(i)} for i in range(count - 2) + ] + value.update(logical_bytes=count, payload_bytes=1) + backup_manifest.validate(value) + assert len(value["files"]) + len(value["directories"]) == 98304 + assert backup_manifest.MAX_BYTES == 7 * 1024**3 + assert backup_manifest.MAX_LOGICAL_BYTES == 12 * 1024**3 + old = {**value, "schema_version": 3, "format": backup_manifest.FORMAT, "payload_bytes": count} + old.pop("logical_bytes") + with pytest.raises(policy.DeliveryError, match="inventory count exceeds bound"): + backup_manifest.validate(old) + value["files"].append({**template, "path": "releases/one-too-many"}) + value["logical_bytes"] += 1 + with pytest.raises(policy.DeliveryError, match="inventory count exceeds bound"): + backup_manifest.validate(value) + + +def test_private_metadata_rejects_a_file_above_the_explicit_byte_limit(tmp_path): + path = tmp_path / "metadata.json" + path.write_bytes(b" " * (backup_manifest.MAX_METADATA_BYTES + 1)) + with pytest.raises(policy.DeliveryError, match="metadata byte limit"): + backup_manifest.read_metadata(path) diff --git a/backend/tests/test_delivery_restore_io.py b/backend/tests/test_delivery_restore_io.py index e886456..723eab7 100644 --- a/backend/tests/test_delivery_restore_io.py +++ b/backend/tests/test_delivery_restore_io.py @@ -509,3 +509,29 @@ def test_grouped_real_copy_workers_join_and_return_independent_verified_files( path, uid, gid, mode = destination(row) assert path.read_bytes() == b"shared" restore.unchanged_file(path, result[row["path"]], uid, gid, mode) + + +@pytest.mark.parametrize( + "name", ["releases/../private", "factory//requests", "unknown/file", "releases/.ssh/key"] +) +def test_untrusted_restore_target_names_still_require_complete_validation(tmp_path, name): + record = {"path": name, "uid": 0, "gid": 0, "mode": 0o600, "bytes": 0} + with pytest.raises(DeliveryError, match="private backup member"): + restore.target(record, tmp_path) + + +def test_pending_copy_refuses_a_dangling_symlink_without_creating_its_target(tmp_path): + source = tmp_path / "source" + source.write_bytes(b"trusted bytes") + target = tmp_path / "target" + missing = tmp_path / "missing" + target.symlink_to(missing) + item = { + "bytes": source.stat().st_size, + "sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + } + with pytest.raises(DeliveryError, match="existing recovery destination differs"): + restore_io.pending_file( + source, target, item, os.getuid(), os.getgid(), 0o600, time.monotonic() + 10 + ) + assert target.is_symlink() and not missing.exists() diff --git a/scripts/release/README.md b/scripts/release/README.md index 1f44f2a..f5a94a3 100644 --- a/scripts/release/README.md +++ b/scripts/release/README.md @@ -421,6 +421,15 @@ Every target's inode, ownership, mode, link count, size and nanosecond timestamp must remain unchanged until the final receipt. These observations are local to one stopped-node recovery and are never reused after an interrupted attempt. Directory entries are flushed after their children before completion is recorded. +The private v4 inventory accepts up to 98304 logical file/directory entries +and 20 MiB of metadata so retained software/runtime versions are included in full. +The 12 GiB logical, 7 GiB physical and 256 MiB individual-file limits remain; +capture keeps its 256 MiB memory, 50% CPU and original finite deadline. Version 3 +keeps its 65536-entry bound. All normalized directory nodes must have a declared +direct parent, which also verifies every ancestor of every file. During restoration, +only this completely verified inventory can reuse its checked target names; +untrusted calls still validate paths. These bounds require actual capture and +recovery validation on the target node; exceeding them is an explicit refusal. The cold operator, activation and immutable service/dependency/journal profile installation can reuse a public bundle's complete byte verification within one synchronous operation. Every file and directory identity, owner, mode, link count, diff --git a/scripts/release/backup_manifest.py b/scripts/release/backup_manifest.py index b4b869a..050f098 100644 --- a/scripts/release/backup_manifest.py +++ b/scripts/release/backup_manifest.py @@ -23,8 +23,8 @@ MAX_FILE_BYTES = 256 * 1024**2 MAX_ENTRIES = 65536 # v4 preserves logical versions independently of their deduplicated storage. MAX_LOGICAL_BYTES = 12 * 1024**3 -MAX_LOGICAL_ENTRIES = 81920 -MAX_METADATA_BYTES = 16 * 1024**2 +MAX_LOGICAL_ENTRIES = 98304 +MAX_METADATA_BYTES = 20 * 1024**2 DOMAINS = { "postgres", "artifacts", @@ -279,10 +279,13 @@ def validate(value: dict) -> dict: directories = {item["path"] for item in dirs} if not DOMAINS <= directories: raise DeliveryError("private backup omitted a required source domain") + # Every normalized file and directory is in seen. Requiring the direct + # parent of every node proves the whole ancestor chain transitively, + # including ancestors of directories, without rebuilding Path chains. for path in seen: - for parent in Path(path).parents: - if parent.as_posix() != "." and parent.as_posix() not in directories: - raise DeliveryError("private backup parent missing") + parent = path.rpartition("/")[0] + if parent and parent not in directories: + raise DeliveryError("private backup parent missing") lookup = {item["path"]: item for item in files} for reference in value["artifact_references"]["objects"]: item = lookup.get("artifacts/" + reference["digest"]) diff --git a/scripts/release/restore.py b/scripts/release/restore.py index ecca58b..687e113 100644 --- a/scripts/release/restore.py +++ b/scripts/release/restore.py @@ -122,7 +122,11 @@ def stopped() -> None: def target(record: dict, recovery: Path) -> tuple[Path, int, int, int]: - name = backup_manifest.name(record["path"]) + return _target(backup_manifest.name(record["path"]), record, recovery) + + +def _target(name: str, record: dict, recovery: Path) -> tuple[Path, int, int, int]: + """Map a name already checked by target() or the complete backup verifier.""" domain, _, suffix = name.partition("/") if domain in LIVE: path = LIVE[domain] / suffix if suffix else LIVE[domain] @@ -367,7 +371,7 @@ def restore(recovery_id: str, backup_id: str, expected: str) -> dict: for item in sorted( value["directories"], key=lambda row: (row["path"].count("/"), row["path"]) ): - path, uid, gid, mode = target(item, recovery) + path, uid, gid, mode = _target(item["path"], item, recovery) if path.exists() or path.is_symlink(): member(path, uid, gid, mode, directory=True) else: @@ -381,7 +385,7 @@ def restore(recovery_id: str, backup_id: str, expected: str) -> dict: raise DeliveryError("cold recovery destination reserve insufficient") def copy(item: dict) -> tuple[int, ...]: - path, uid, gid, mode = target(item, recovery) + path, uid, gid, mode = _target(item["path"], item, recovery) return write_file( backup_manifest.member_path(root, value, item), path, @@ -397,7 +401,7 @@ def restore(recovery_id: str, backup_id: str, expected: str) -> dict: verified = restore_io.materialize( value["files"], lambda item: backup_manifest.member_path(root, value, item), - lambda item: target(item, recovery), + lambda item: _target(item["path"], item, recovery), readback, filesystems, deadline, @@ -412,7 +416,7 @@ def restore(recovery_id: str, backup_id: str, expected: str) -> dict: for item in sorted( value["directories"], key=lambda row: row["path"].count("/"), reverse=True ): - path, uid, gid, mode = target(item, recovery) + path, uid, gid, mode = _target(item["path"], item, recovery) current = path.lstat() if (current.st_uid, current.st_gid) != (uid, gid): os.chown(path, uid, gid, follow_symlinks=False) @@ -427,7 +431,7 @@ def restore(recovery_id: str, backup_id: str, expected: str) -> dict: assert filesystems is not None filesystems.sync(deadline) for item in value["directories"]: - path, uid, gid, mode = target(item, recovery) + path, uid, gid, mode = _target(item["path"], item, recovery) member(path, uid, gid, mode, directory=True) else: install.sync_directory(recovery / "history") @@ -438,7 +442,7 @@ def restore(recovery_id: str, backup_id: str, expected: str) -> dict: for item in value["files"]: if time.monotonic() >= deadline: raise DeliveryError("recovery verification deadline exceeded") - path, uid, gid, mode = target(item, recovery) + path, uid, gid, mode = _target(item["path"], item, recovery) unchanged_file(path, verified[item["path"]], uid, gid, mode) if filesystems is not None: filesystems.close() diff --git a/scripts/release/restore_io.py b/scripts/release/restore_io.py index 3de3cff..a39a834 100644 --- a/scripts/release/restore_io.py +++ b/scripts/release/restore_io.py @@ -120,8 +120,11 @@ def pending_file( ) -> tuple[int, ...]: """Materialize only; this witness cannot certify bytes before batch readback.""" remaining(deadline) - if path.exists() or path.is_symlink(): + try: info = path.lstat() + except FileNotFoundError: + info = None + if info is not None: if ( not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 @@ -143,9 +146,17 @@ def pending_file( ): file_copy.transfer(incoming, outgoing, item["bytes"], deadline=deadline) outgoing.flush() - os.fchown(outgoing.fileno(), uid, gid) - os.fchmod(outgoing.fileno(), mode) - witness = file_copy._identity(os.fstat(outgoing.fileno())) + current = os.fstat(outgoing.fileno()) + changed = False + if (current.st_uid, current.st_gid) != (uid, gid): + os.fchown(outgoing.fileno(), uid, gid) + changed = True + # fchown may clear executable/set-ID bits; always restore the requested + # allowed mode after an ownership change, even if it matched before. + if changed or stat.S_IMODE(current.st_mode) != mode: + os.fchmod(outgoing.fileno(), mode) + changed = True + witness = file_copy._identity(os.fstat(outgoing.fileno()) if changed else current) if file_copy._identity(path.lstat()) != witness: raise DeliveryError("recovery destination changed before synchronization") return witness