perf(delivery): streamline complete recovery inventories

backend/tests/test_delivery_backup.py

@@ -302,3 +302,48 @@ def test_delayed_filesystem_write_error_is_not_success(tmp_path, monkeypatch):
     monkeypatch.setattr(backup_objects.ctypes, "CDLL", lambda *_, **__: Library())
     with pytest.raises(policy.DeliveryError, match="filesystem synchronization failed"):
         backup_objects.sync_filesystem(tmp_path, time.monotonic() + 10)
+
+
+@pytest.mark.parametrize("missing", ["releases/older", "releases/older/nested"])
+def test_every_directory_ancestor_is_required_even_when_file_parent_exists(snapshot, missing):
+    value = object_format(snapshot)
+    value["directories"] = value["directories"] + [
+        {"path": path, "uid": 0, "gid": 0, "mode": 0o700}
+        for path in ("releases/older", "releases/older/nested")
+        if path != missing
+    ]
+    value["files"] = [
+        {**row, "path": "releases/older/nested/file"} if row["path"] == "releases/first" else row
+        for row in value["files"]
+    ]
+    with pytest.raises(policy.DeliveryError, match="private backup parent missing"):
+        backup_manifest.validate(value)
+
+
+def test_v4_complete_version_inventory_keeps_an_explicit_logical_entry_bound(snapshot):
+    value = object_format(snapshot)
+    count = backup_manifest.MAX_LOGICAL_ENTRIES - len(value["directories"])
+    template = {**value["files"][0], "bytes": 1, "sha256": hashlib.sha256(b"x").hexdigest()}
+    value["files"] = [{**template, "path": row["path"]} for row in value["files"][:2]] + [
+        {**template, "path": "releases/item-" + str(i)} for i in range(count - 2)
+    ]
+    value.update(logical_bytes=count, payload_bytes=1)
+    backup_manifest.validate(value)
+    assert len(value["files"]) + len(value["directories"]) == 98304
+    assert backup_manifest.MAX_BYTES == 7 * 1024**3
+    assert backup_manifest.MAX_LOGICAL_BYTES == 12 * 1024**3
+    old = {**value, "schema_version": 3, "format": backup_manifest.FORMAT, "payload_bytes": count}
+    old.pop("logical_bytes")
+    with pytest.raises(policy.DeliveryError, match="inventory count exceeds bound"):
+        backup_manifest.validate(old)
+    value["files"].append({**template, "path": "releases/one-too-many"})
+    value["logical_bytes"] += 1
+    with pytest.raises(policy.DeliveryError, match="inventory count exceeds bound"):
+        backup_manifest.validate(value)
+
+
+def test_private_metadata_rejects_a_file_above_the_explicit_byte_limit(tmp_path):
+    path = tmp_path / "metadata.json"
+    path.write_bytes(b" " * (backup_manifest.MAX_METADATA_BYTES + 1))
+    with pytest.raises(policy.DeliveryError, match="metadata byte limit"):
+        backup_manifest.read_metadata(path)

backend/tests/test_delivery_restore_io.py

@@ -509,3 +509,29 @@ def test_grouped_real_copy_workers_join_and_return_independent_verified_files(
         path, uid, gid, mode = destination(row)
         assert path.read_bytes() == b"shared"
         restore.unchanged_file(path, result[row["path"]], uid, gid, mode)
+
+
+@pytest.mark.parametrize(
+    "name", ["releases/../private", "factory//requests", "unknown/file", "releases/.ssh/key"]
+)
+def test_untrusted_restore_target_names_still_require_complete_validation(tmp_path, name):
+    record = {"path": name, "uid": 0, "gid": 0, "mode": 0o600, "bytes": 0}
+    with pytest.raises(DeliveryError, match="private backup member"):
+        restore.target(record, tmp_path)
+
+
+def test_pending_copy_refuses_a_dangling_symlink_without_creating_its_target(tmp_path):
+    source = tmp_path / "source"
+    source.write_bytes(b"trusted bytes")
+    target = tmp_path / "target"
+    missing = tmp_path / "missing"
+    target.symlink_to(missing)
+    item = {
+        "bytes": source.stat().st_size,
+        "sha256": hashlib.sha256(source.read_bytes()).hexdigest(),
+    }
+    with pytest.raises(DeliveryError, match="existing recovery destination differs"):
+        restore_io.pending_file(
+            source, target, item, os.getuid(), os.getgid(), 0o600, time.monotonic() + 10
+        )
+    assert target.is_symlink() and not missing.exists()

scripts/release/README.md

@@ -421,6 +421,15 @@ Every target's inode, ownership, mode, link count, size and nanosecond timestamp
 must remain unchanged until the final receipt. These observations are local to
 one stopped-node recovery and are never reused after an interrupted attempt.
 Directory entries are flushed after their children before completion is recorded.
+The private v4 inventory accepts up to 98304 logical file/directory entries
+and 20 MiB of metadata so retained software/runtime versions are included in full.
+The 12 GiB logical, 7 GiB physical and 256 MiB individual-file limits remain;
+capture keeps its 256 MiB memory, 50% CPU and original finite deadline. Version 3
+keeps its 65536-entry bound. All normalized directory nodes must have a declared
+direct parent, which also verifies every ancestor of every file. During restoration,
+only this completely verified inventory can reuse its checked target names;
+untrusted calls still validate paths. These bounds require actual capture and
+recovery validation on the target node; exceeding them is an explicit refusal.
 The cold operator, activation and immutable service/dependency/journal profile
 installation can reuse a public bundle's complete byte verification within one
 synchronous operation. Every file and directory identity, owner, mode, link count,

scripts/release/backup_manifest.py

@@ -23,8 +23,8 @@ MAX_FILE_BYTES = 256 * 1024**2
 MAX_ENTRIES = 65536
 # v4 preserves logical versions independently of their deduplicated storage.
 MAX_LOGICAL_BYTES = 12 * 1024**3
-MAX_LOGICAL_ENTRIES = 81920
-MAX_METADATA_BYTES = 16 * 1024**2
+MAX_LOGICAL_ENTRIES = 98304
+MAX_METADATA_BYTES = 20 * 1024**2
 DOMAINS = {
     "postgres",
     "artifacts",
@@ -279,10 +279,13 @@ def validate(value: dict) -> dict:
     directories = {item["path"] for item in dirs}
     if not DOMAINS <= directories:
         raise DeliveryError("private backup omitted a required source domain")
+    # Every normalized file and directory is in seen. Requiring the direct
+    # parent of every node proves the whole ancestor chain transitively,
+    # including ancestors of directories, without rebuilding Path chains.
     for path in seen:
-        for parent in Path(path).parents:
-            if parent.as_posix() != "." and parent.as_posix() not in directories:
-                raise DeliveryError("private backup parent missing")
+        parent = path.rpartition("/")[0]
+        if parent and parent not in directories:
+            raise DeliveryError("private backup parent missing")
     lookup = {item["path"]: item for item in files}
     for reference in value["artifact_references"]["objects"]:
         item = lookup.get("artifacts/" + reference["digest"])

scripts/release/restore.py

@@ -122,7 +122,11 @@ def stopped() -> None:
 
 
 def target(record: dict, recovery: Path) -> tuple[Path, int, int, int]:
-    name = backup_manifest.name(record["path"])
+    return _target(backup_manifest.name(record["path"]), record, recovery)
+
+
+def _target(name: str, record: dict, recovery: Path) -> tuple[Path, int, int, int]:
+    """Map a name already checked by target() or the complete backup verifier."""
     domain, _, suffix = name.partition("/")
     if domain in LIVE:
         path = LIVE[domain] / suffix if suffix else LIVE[domain]
@@ -367,7 +371,7 @@ def restore(recovery_id: str, backup_id: str, expected: str) -> dict:
         for item in sorted(
             value["directories"], key=lambda row: (row["path"].count("/"), row["path"])
         ):
-            path, uid, gid, mode = target(item, recovery)
+            path, uid, gid, mode = _target(item["path"], item, recovery)
             if path.exists() or path.is_symlink():
                 member(path, uid, gid, mode, directory=True)
             else:
@@ -381,7 +385,7 @@ def restore(recovery_id: str, backup_id: str, expected: str) -> dict:
             raise DeliveryError("cold recovery destination reserve insufficient")
 
         def copy(item: dict) -> tuple[int, ...]:
-            path, uid, gid, mode = target(item, recovery)
+            path, uid, gid, mode = _target(item["path"], item, recovery)
             return write_file(
                 backup_manifest.member_path(root, value, item),
                 path,
@@ -397,7 +401,7 @@ def restore(recovery_id: str, backup_id: str, expected: str) -> dict:
             verified = restore_io.materialize(
                 value["files"],
                 lambda item: backup_manifest.member_path(root, value, item),
-                lambda item: target(item, recovery),
+                lambda item: _target(item["path"], item, recovery),
                 readback,
                 filesystems,
                 deadline,
@@ -412,7 +416,7 @@ def restore(recovery_id: str, backup_id: str, expected: str) -> dict:
         for item in sorted(
             value["directories"], key=lambda row: row["path"].count("/"), reverse=True
         ):
-            path, uid, gid, mode = target(item, recovery)
+            path, uid, gid, mode = _target(item["path"], item, recovery)
             current = path.lstat()
             if (current.st_uid, current.st_gid) != (uid, gid):
                 os.chown(path, uid, gid, follow_symlinks=False)
@@ -427,7 +431,7 @@ def restore(recovery_id: str, backup_id: str, expected: str) -> dict:
             assert filesystems is not None
             filesystems.sync(deadline)
             for item in value["directories"]:
-                path, uid, gid, mode = target(item, recovery)
+                path, uid, gid, mode = _target(item["path"], item, recovery)
                 member(path, uid, gid, mode, directory=True)
         else:
             install.sync_directory(recovery / "history")
@@ -438,7 +442,7 @@ def restore(recovery_id: str, backup_id: str, expected: str) -> dict:
         for item in value["files"]:
             if time.monotonic() >= deadline:
                 raise DeliveryError("recovery verification deadline exceeded")
-            path, uid, gid, mode = target(item, recovery)
+            path, uid, gid, mode = _target(item["path"], item, recovery)
             unchanged_file(path, verified[item["path"]], uid, gid, mode)
         if filesystems is not None:
             filesystems.close()

scripts/release/restore_io.py

@@ -120,8 +120,11 @@ def pending_file(
 ) -> tuple[int, ...]:
     """Materialize only; this witness cannot certify bytes before batch readback."""
     remaining(deadline)
-    if path.exists() or path.is_symlink():
+    try:
         info = path.lstat()
+    except FileNotFoundError:
+        info = None
+    if info is not None:
         if (
             not stat.S_ISREG(info.st_mode)
             or info.st_nlink != 1
@@ -143,9 +146,17 @@ def pending_file(
     ):
         file_copy.transfer(incoming, outgoing, item["bytes"], deadline=deadline)
         outgoing.flush()
-        os.fchown(outgoing.fileno(), uid, gid)
-        os.fchmod(outgoing.fileno(), mode)
-        witness = file_copy._identity(os.fstat(outgoing.fileno()))
+        current = os.fstat(outgoing.fileno())
+        changed = False
+        if (current.st_uid, current.st_gid) != (uid, gid):
+            os.fchown(outgoing.fileno(), uid, gid)
+            changed = True
+        # fchown may clear executable/set-ID bits; always restore the requested
+        # allowed mode after an ownership change, even if it matched before.
+        if changed or stat.S_IMODE(current.st_mode) != mode:
+            os.fchmod(outgoing.fileno(), mode)
+            changed = True
+        witness = file_copy._identity(os.fstat(outgoing.fileno()) if changed else current)
     if file_copy._identity(path.lstat()) != witness:
         raise DeliveryError("recovery destination changed before synchronization")
     return witness