perf(delivery): batch installs and parallelize cold recovery

backend/tests/test_delivery.py

@@ -365,7 +365,28 @@ def test_changed_copy_retains_incomplete_file_and_never_overwrites_it(tmp_path,
     assert target.read_bytes() == b"changed"
 
 
-def test_frozen_copy_reuses_verified_records_and_checks_full_destination(tmp_path, monkeypatch):
+@pytest.fixture
+def frozen_filesystem_model(tmp_path, monkeypatch):
+    """Only fsync tiny owned fixtures; real filesystem flush remains VM-only."""
+
+    class Sync:
+        def __call__(self, descriptor):
+            assert stat.S_ISDIR(os.fstat(descriptor).st_mode)
+            for path in tmp_path.rglob("*"):
+                if path.is_file():
+                    assert path.stat().st_uid == os.getuid() and path.stat().st_size < 4096
+                    with path.open("rb") as stream:
+                        os.fsync(stream.fileno())
+            return 0
+
+    monkeypatch.setattr(
+        installer.restore_io.ctypes, "CDLL", lambda *_a, **_kw: SimpleNamespace(syncfs=Sync())
+    )
+
+
+def test_frozen_copy_reuses_verified_records_and_checks_full_destination(
+    tmp_path, monkeypatch, frozen_filesystem_model
+):
     monkeypatch.setattr(installer.preflight, "protected", lambda *args, **kwargs: None)
     source, target = tmp_path / "source", tmp_path / "target"
     source.mkdir()
@@ -390,7 +411,9 @@ def test_frozen_copy_reuses_verified_records_and_checks_full_destination(tmp_pat
     assert scanned == [target, target]
 
 
-def test_frozen_copy_rejects_destination_io_mismatch_before_acceptance(tmp_path, monkeypatch):
+def test_frozen_copy_rejects_destination_io_mismatch_before_acceptance(
+    tmp_path, monkeypatch, frozen_filesystem_model
+):
     # Simulated I/O mismatch in an ordinary owned fixture; Root protection and
     # actual disk/error isolation are still separate VM acceptance.
     monkeypatch.setattr(installer.preflight, "protected", lambda *args, **kwargs: None)
@@ -416,7 +439,9 @@ def test_frozen_copy_rejects_destination_io_mismatch_before_acceptance(tmp_path,
         installer.frozen_copy(source, target, verified_records=records)
 
 
-def test_frozen_copy_rejects_changed_source_against_trusted_records(tmp_path, monkeypatch):
+def test_frozen_copy_rejects_changed_source_against_trusted_records(
+    tmp_path, monkeypatch, frozen_filesystem_model
+):
     # Ordinary fixture bytes exercise the deferred batch validation. Actual
     # Root protection and installation are separately verified in the VM.
     monkeypatch.setattr(installer.preflight, "protected", lambda *args, **kwargs: None)
@@ -439,6 +464,30 @@ def test_frozen_copy_rejects_changed_source_against_trusted_records(tmp_path, mo
         installer.frozen_copy(source, target, verified_records=records)
 
 
+def test_frozen_copy_rejects_sync_failure_even_when_destination_bytes_match(tmp_path, monkeypatch):
+    class FailedSync:
+        def __call__(self, _descriptor):
+            return -1
+
+    monkeypatch.setattr(installer.preflight, "protected", lambda *args, **kwargs: None)
+    monkeypatch.setattr(
+        installer.restore_io.ctypes,
+        "CDLL",
+        lambda *_a, **_kw: SimpleNamespace(syncfs=FailedSync()),
+    )
+    source, target = tmp_path / "source", tmp_path / "target"
+    source.mkdir()
+    (source / "backend").mkdir()
+    payload = source / "backend/ordinary.txt"
+    payload.write_bytes(b"known ordinary bytes")
+    payload.chmod(0o444)
+    (source / "backend").chmod(0o555)
+    records = manifest.inventory(source)
+    with pytest.raises(policy.DeliveryError, match="filesystem synchronization failed"):
+        installer.frozen_copy(source, target, verified_records=records)
+    assert (target / "backend/ordinary.txt").read_bytes() == payload.read_bytes()
+
+
 def test_kernel_observation_keeps_cumulative_counters_and_rejects_duplicate_rows():
     raw = (
         "usage_usec 17\nuser_usec 11\nsystem_usec 6\nnr_periods 9\nnr_throttled 2\nthrottled_usec 3"

backend/tests/test_delivery_restore_io.py

@@ -2,6 +2,7 @@
 
 import errno
 import hashlib
+import multiprocessing
 import os
 import stat
 import time
@@ -65,6 +66,19 @@ def assert_closed(descriptors):
         assert failure.value.errno == errno.EBADF
 
 
+class ReadAfterSync:
+    """Picklable observation of actual tiny fixture reads in the child processes."""
+
+    def __init__(self, synced, reads, descriptors):
+        self.synced, self.reads, self.descriptors = synced, reads, descriptors
+
+    def __call__(self, path, item, uid, gid, mode, deadline):
+        assert self.synced.read_bytes() == b"sync completed"
+        assert all(os.fstat(fd).st_ino > 0 for fd in self.descriptors)
+        (self.reads / item["path"]).touch()
+        return restore.readback(path, item, uid, gid, mode, deadline)
+
+
 def test_bounded_batches_group_identical_objects_without_losing_paths():
     rows = [
         {"path": str(index), "sha256": str(index % 2), "bytes": 1}
@@ -89,22 +103,31 @@ def test_repeated_objects_expand_to_independent_durable_verified_destinations(
     rows, source, destination = fixture_rows(tmp_path, [b"first", b"other", b"first", b"first"])
     events = []
     target = destination(rows[0])[0].parent
+    synced, reads = tmp_path / "synced", tmp_path / "reads"
+    reads.mkdir()
     with closing(filesystem_model(monkeypatch, target, events)) as filesystems:
         descriptors = [descriptor for _, descriptor, _ in filesystems.handles.values()]
         assert list(target.iterdir()) == []
 
-        def readback(path, item, uid, gid, mode, deadline):
-            assert events and events[0][0] == "sync"
-            assert all(os.fstat(fd).st_ino > 0 for fd in descriptors)
-            events.append(("read", item["path"]))
-            return restore.readback(path, item, uid, gid, mode, deadline)
+        original_sync = filesystems.sync
+
+        def sync(deadline):
+            original_sync(deadline)
+            synced.write_bytes(b"sync completed")
+
+        monkeypatch.setattr(filesystems, "sync", sync)
 
         verified = restore_io.materialize(
-            rows, source, destination, readback, filesystems, time.monotonic() + 120
+            rows,
+            source,
+            destination,
+            ReadAfterSync(synced, reads, descriptors),
+            filesystems,
+            time.monotonic() + 120,
         )
     assert_closed(descriptors)
     assert set(verified) == {row["path"] for row in rows}
-    assert len([event for event in events if event[0] == "read"]) == len(rows)
+    assert len(list(reads.iterdir())) == len(rows)
     inodes = set()
     for row in rows:
         path, uid, gid, mode = destination(row)
@@ -117,6 +140,7 @@ def test_repeated_objects_expand_to_independent_durable_verified_destinations(
 
 
 def test_existing_wrong_bytes_are_refused_without_overwrite(tmp_path, monkeypatch):
+    prior_children = {child.pid for child in multiprocessing.active_children()}
     rows, source, destination = fixture_rows(tmp_path, [b"new"])
     path, _, _, mode = destination(rows[0])
     path.write_bytes(b"bad")
@@ -128,6 +152,7 @@ def test_existing_wrong_bytes_are_refused_without_overwrite(tmp_path, monkeypatc
                 rows, source, destination, restore.readback, filesystems, time.monotonic() + 120
             )
     assert path.read_bytes() == b"bad" and restore.file_witness(path.stat()) == before
+    assert {child.pid for child in multiprocessing.active_children()} == prior_children
 
 
 def test_failed_sync_keeps_private_files_and_cannot_return_verified_witnesses(
@@ -259,3 +284,46 @@ def test_constructor_closes_earlier_cursor_when_later_anchor_is_invalid(tmp_path
         restore_io.Filesystems([valid, invalid], time.monotonic() + 120)
     assert len(opened) == 1
     assert_closed(opened)
+
+
+def test_actual_two_copy_processes_join_before_return_and_preserve_all_bytes(tmp_path, monkeypatch):
+    rows, source, destination = fixture_rows(
+        tmp_path, [b"known first", b"known second", b"known third", b"known fourth"]
+    )
+    prior_children = {child.pid for child in multiprocessing.active_children()}
+    markers = tmp_path / "worker-identities"
+    markers.mkdir()
+    original = restore_io.pending_file
+
+    def record_worker(*args):
+        (markers / str(os.getpid())).touch()
+        deadline = time.monotonic() + 5
+        while len(list(markers.iterdir())) < 2:
+            assert time.monotonic() < deadline
+            time.sleep(0.01)
+        return original(*args)
+
+    monkeypatch.setattr(restore_io, "pending_file", record_worker)
+    with closing(filesystem_model(monkeypatch, destination(rows[0])[0].parent, [])) as fs:
+        verified = restore_io.materialize(
+            rows, source, destination, restore.readback, fs, time.monotonic() + 120
+        )
+    assert len(list(markers.iterdir())) == 2
+    assert {child.pid for child in multiprocessing.active_children()} == prior_children
+    assert all(not (restore_io.Path("/proc") / p.name).exists() for p in markers.iterdir())
+    assert set(verified) == {row["path"] for row in rows}
+    for row in rows:
+        path, uid, gid, mode = destination(row)
+        assert path.read_bytes() == source(row).read_bytes()
+        restore.unchanged_file(path, verified[row["path"]], uid, gid, mode)
+
+
+def test_process_copy_refuses_fork_from_a_multithreaded_owner(tmp_path, monkeypatch):
+    rows, source, destination = fixture_rows(tmp_path, [b"known fixture"])
+    monkeypatch.setattr(restore_io.threading, "active_count", lambda: 2)
+    with closing(filesystem_model(monkeypatch, destination(rows[0])[0].parent, [])) as fs:
+        with pytest.raises(DeliveryError, match="one POSIX owner thread"):
+            restore_io.materialize(
+                rows, source, destination, restore.readback, fs, time.monotonic() + 120
+            )
+    assert not destination(rows[0])[0].exists()