perf(delivery): batch durable cold recovery copies
This commit is contained in:
@@ -0,0 +1,261 @@
|
||||
"""Trusted ordinary byte fixtures; real Root/cgroup/syncfs/RTO remains VM-only."""
|
||||
|
||||
import errno
|
||||
import hashlib
|
||||
import os
|
||||
import stat
|
||||
import time
|
||||
from contextlib import closing
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from scripts.release import restore, restore_io
|
||||
from scripts.release.policy import DeliveryError
|
||||
|
||||
|
||||
def fixture_rows(tmp_path, contents):
|
||||
source = tmp_path / "source"
|
||||
target = tmp_path / "target"
|
||||
source.mkdir()
|
||||
target.mkdir()
|
||||
rows = []
|
||||
for index, raw in enumerate(contents):
|
||||
digest = hashlib.sha256(raw).hexdigest()
|
||||
path = source / digest
|
||||
if not path.exists():
|
||||
path.write_bytes(raw)
|
||||
rows.append({"path": str(index), "bytes": len(raw), "sha256": digest})
|
||||
return (
|
||||
rows,
|
||||
lambda row: source / row["sha256"],
|
||||
lambda row: (
|
||||
target / row["path"],
|
||||
os.getuid(),
|
||||
os.getgid(),
|
||||
0o600 if int(row["path"]) % 2 else 0o400,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def filesystem_model(monkeypatch, target, events, callback=None):
|
||||
"""Model syncfs using only tiny owned fixture fsyncs; never host syncfs."""
|
||||
|
||||
class Sync:
|
||||
def __call__(self, descriptor):
|
||||
assert stat.S_ISDIR(os.fstat(descriptor).st_mode)
|
||||
if callback is not None:
|
||||
callback()
|
||||
files = [p for p in target.iterdir() if p.is_file()]
|
||||
for path in files:
|
||||
with path.open("rb") as stream:
|
||||
os.fsync(stream.fileno())
|
||||
events.append(("sync", descriptor))
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(
|
||||
restore_io.ctypes, "CDLL", lambda *_a, **_kw: SimpleNamespace(syncfs=Sync())
|
||||
)
|
||||
return restore_io.Filesystems([target], time.monotonic() + 120)
|
||||
|
||||
|
||||
def assert_closed(descriptors):
|
||||
for descriptor in descriptors:
|
||||
with pytest.raises(OSError) as failure:
|
||||
os.fstat(descriptor)
|
||||
assert failure.value.errno == errno.EBADF
|
||||
|
||||
|
||||
def test_bounded_batches_group_identical_objects_without_losing_paths():
|
||||
rows = [
|
||||
{"path": str(index), "sha256": str(index % 2), "bytes": 1}
|
||||
for index in range(restore_io.BATCH_FILES + 4)
|
||||
]
|
||||
rows.append({"path": "large", "sha256": "2", "bytes": restore_io.BATCH_BYTES + 1})
|
||||
groups = list(restore_io.batches(rows))
|
||||
flattened = [row for group in groups for row in group]
|
||||
assert {row["path"] for row in flattened} == {row["path"] for row in rows}
|
||||
assert len(flattened) == len(rows)
|
||||
assert [row["sha256"] for row in flattened] == sorted(row["sha256"] for row in rows)
|
||||
assert all(len(group) <= restore_io.BATCH_FILES for group in groups)
|
||||
assert all(
|
||||
sum(row["bytes"] for row in group) <= restore_io.BATCH_BYTES or len(group) == 1
|
||||
for group in groups
|
||||
)
|
||||
|
||||
|
||||
def test_repeated_objects_expand_to_independent_durable_verified_destinations(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
rows, source, destination = fixture_rows(tmp_path, [b"first", b"other", b"first", b"first"])
|
||||
events = []
|
||||
target = destination(rows[0])[0].parent
|
||||
with closing(filesystem_model(monkeypatch, target, events)) as filesystems:
|
||||
descriptors = [descriptor for _, descriptor, _ in filesystems.handles.values()]
|
||||
assert list(target.iterdir()) == []
|
||||
|
||||
def readback(path, item, uid, gid, mode, deadline):
|
||||
assert events and events[0][0] == "sync"
|
||||
assert all(os.fstat(fd).st_ino > 0 for fd in descriptors)
|
||||
events.append(("read", item["path"]))
|
||||
return restore.readback(path, item, uid, gid, mode, deadline)
|
||||
|
||||
verified = restore_io.materialize(
|
||||
rows, source, destination, readback, filesystems, time.monotonic() + 120
|
||||
)
|
||||
assert_closed(descriptors)
|
||||
assert set(verified) == {row["path"] for row in rows}
|
||||
assert len([event for event in events if event[0] == "read"]) == len(rows)
|
||||
inodes = set()
|
||||
for row in rows:
|
||||
path, uid, gid, mode = destination(row)
|
||||
assert path.read_bytes() == source(row).read_bytes()
|
||||
assert stat.S_IMODE(path.stat().st_mode) == mode
|
||||
assert path.stat().st_nlink == 1
|
||||
inodes.add(path.stat().st_ino)
|
||||
restore.unchanged_file(path, verified[row["path"]], uid, gid, mode)
|
||||
assert len(inodes) == len(rows)
|
||||
|
||||
|
||||
def test_existing_wrong_bytes_are_refused_without_overwrite(tmp_path, monkeypatch):
|
||||
rows, source, destination = fixture_rows(tmp_path, [b"new"])
|
||||
path, _, _, mode = destination(rows[0])
|
||||
path.write_bytes(b"bad")
|
||||
path.chmod(mode)
|
||||
before = restore.file_witness(path.stat())
|
||||
with closing(filesystem_model(monkeypatch, path.parent, [])) as filesystems:
|
||||
with pytest.raises(DeliveryError, match="byte verification"):
|
||||
restore_io.materialize(
|
||||
rows, source, destination, restore.readback, filesystems, time.monotonic() + 120
|
||||
)
|
||||
assert path.read_bytes() == b"bad" and restore.file_witness(path.stat()) == before
|
||||
|
||||
|
||||
def test_failed_sync_keeps_private_files_and_cannot_return_verified_witnesses(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
rows, source, destination = fixture_rows(tmp_path, [b"a", b"b", b"c"])
|
||||
reads = []
|
||||
|
||||
def failed_sync():
|
||||
raise DeliveryError("synthetic synchronization failure")
|
||||
|
||||
with closing(
|
||||
filesystem_model(monkeypatch, destination(rows[0])[0].parent, [], failed_sync)
|
||||
) as filesystems:
|
||||
descriptors = [fd for _, fd, _ in filesystems.handles.values()]
|
||||
with pytest.raises(DeliveryError, match="synthetic synchronization failure"):
|
||||
restore_io.materialize(
|
||||
rows,
|
||||
source,
|
||||
destination,
|
||||
lambda *args: reads.append(args),
|
||||
filesystems,
|
||||
time.monotonic() + 120,
|
||||
)
|
||||
assert_closed(descriptors)
|
||||
assert reads == []
|
||||
assert all(destination(row)[0].is_file() for row in rows)
|
||||
|
||||
|
||||
def test_same_bytes_inode_replacement_across_sync_cannot_adopt_new_file(tmp_path, monkeypatch):
|
||||
rows, source, destination = fixture_rows(tmp_path, [b"same"])
|
||||
|
||||
def replace_inode():
|
||||
path, _, _, mode = destination(rows[0])
|
||||
replacement = path.parent / "replacement"
|
||||
replacement.write_bytes(path.read_bytes())
|
||||
replacement.chmod(mode)
|
||||
replacement.replace(path)
|
||||
|
||||
with closing(
|
||||
filesystem_model(monkeypatch, destination(rows[0])[0].parent, [], replace_inode)
|
||||
) as filesystems:
|
||||
with pytest.raises(DeliveryError, match="changed across synchronization"):
|
||||
restore_io.materialize(
|
||||
rows, source, destination, restore.readback, filesystems, time.monotonic() + 120
|
||||
)
|
||||
|
||||
|
||||
def test_expired_batch_deadline_copies_no_files(tmp_path, monkeypatch):
|
||||
rows, source, destination = fixture_rows(tmp_path, [b"data"])
|
||||
with closing(filesystem_model(monkeypatch, destination(rows[0])[0].parent, [])) as filesystems:
|
||||
with pytest.raises(DeliveryError, match="deadline exceeded"):
|
||||
restore_io.materialize(
|
||||
rows, source, destination, restore.readback, filesystems, time.monotonic() - 1
|
||||
)
|
||||
assert not destination(rows[0])[0].exists()
|
||||
|
||||
|
||||
def test_syncfs_failure_is_propagated_in_the_ordinary_syscall_model(tmp_path, monkeypatch):
|
||||
class FailedSync:
|
||||
def __call__(self, _descriptor):
|
||||
return -1
|
||||
|
||||
monkeypatch.setattr(
|
||||
restore_io.ctypes, "CDLL", lambda *_a, **_kw: SimpleNamespace(syncfs=FailedSync())
|
||||
)
|
||||
with closing(restore_io.Filesystems([tmp_path], time.monotonic() + 120)) as filesystems:
|
||||
descriptors = [fd for _, fd, _ in filesystems.handles.values()]
|
||||
with pytest.raises(DeliveryError, match="filesystem synchronization failed"):
|
||||
filesystems.sync(time.monotonic() + 120)
|
||||
assert_closed(descriptors)
|
||||
|
||||
|
||||
def test_cursor_is_open_before_first_write_and_held_across_batches(tmp_path, monkeypatch):
|
||||
rows, source, destination = fixture_rows(tmp_path, [b"a", b"b", b"c", b"d", b"e"])
|
||||
target = destination(rows[0])[0].parent
|
||||
monkeypatch.setattr(restore_io, "BATCH_FILES", 2)
|
||||
events = []
|
||||
with closing(filesystem_model(monkeypatch, target, events)) as filesystems:
|
||||
descriptors = [fd for _, fd, _ in filesystems.handles.values()]
|
||||
before = {fd: (os.fstat(fd).st_dev, os.fstat(fd).st_ino) for fd in descriptors}
|
||||
real_write = restore_io.pending_file
|
||||
|
||||
def write(*args):
|
||||
assert {fd: (os.fstat(fd).st_dev, os.fstat(fd).st_ino) for fd in descriptors} == before
|
||||
return real_write(*args)
|
||||
|
||||
monkeypatch.setattr(restore_io, "pending_file", write)
|
||||
verified = restore_io.materialize(
|
||||
rows, source, destination, restore.readback, filesystems, time.monotonic() + 120
|
||||
)
|
||||
assert len(verified) == len(rows)
|
||||
assert len(events) == 3
|
||||
assert {fd for event, fd in events if event == "sync"} == set(descriptors)
|
||||
assert_closed(descriptors)
|
||||
|
||||
|
||||
def test_cursor_can_anchor_before_target_directories_exist(tmp_path, monkeypatch):
|
||||
target = tmp_path / "later" / "nested"
|
||||
|
||||
class Sync:
|
||||
def __call__(self, descriptor):
|
||||
assert os.fstat(descriptor).st_ino == tmp_path.stat().st_ino
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(
|
||||
restore_io.ctypes, "CDLL", lambda *_a, **_kw: SimpleNamespace(syncfs=Sync())
|
||||
)
|
||||
with closing(restore_io.Filesystems([target], time.monotonic() + 120)) as filesystems:
|
||||
target.mkdir(parents=True)
|
||||
filesystems.contains(target / "file")
|
||||
filesystems.sync(time.monotonic() + 120)
|
||||
|
||||
|
||||
def test_constructor_closes_earlier_cursor_when_later_anchor_is_invalid(tmp_path, monkeypatch):
|
||||
valid, invalid = tmp_path / "valid", tmp_path / "invalid"
|
||||
valid.mkdir()
|
||||
invalid.write_bytes(b"ordinary owned fixture")
|
||||
opened = []
|
||||
real_open = restore_io.os.open
|
||||
|
||||
def record_open(*args, **kwargs):
|
||||
descriptor = real_open(*args, **kwargs)
|
||||
opened.append(descriptor)
|
||||
return descriptor
|
||||
|
||||
monkeypatch.setattr(restore_io.os, "open", record_open)
|
||||
with pytest.raises(DeliveryError, match="anchor differs"):
|
||||
restore_io.Filesystems([valid, invalid], time.monotonic() + 120)
|
||||
assert len(opened) == 1
|
||||
assert_closed(opened)
|
||||
Reference in New Issue
Block a user