perf(delivery): batch durable cold recovery copies
This commit is contained in:
@@ -0,0 +1,201 @@
|
||||
"""Bounded cold-copy batches, durable before full destination byte readback."""
|
||||
|
||||
import ctypes
|
||||
import os
|
||||
import stat
|
||||
import time
|
||||
from collections.abc import Callable, Iterator, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from . import file_copy
|
||||
from .policy import DeliveryError
|
||||
|
||||
BATCH_FILES = 256
|
||||
BATCH_BYTES = 64 * 1024**2
|
||||
|
||||
|
||||
def remaining(deadline: float) -> None:
|
||||
if time.monotonic() >= deadline:
|
||||
raise DeliveryError("recovery copy deadline exceeded; private partial target retained")
|
||||
|
||||
|
||||
def batches(rows: Sequence[dict]) -> Iterator[list[dict]]:
|
||||
"""Keep repeated objects adjacent; large files have an exclusive batch."""
|
||||
current: list[dict] = []
|
||||
size = 0
|
||||
for row in sorted(rows, key=lambda item: (item["sha256"], item["path"])):
|
||||
count = row["bytes"]
|
||||
if current and (len(current) >= BATCH_FILES or size + count > BATCH_BYTES):
|
||||
yield current
|
||||
current, size = [], 0
|
||||
current.append(row)
|
||||
size += count
|
||||
if size >= BATCH_BYTES:
|
||||
yield current
|
||||
current, size = [], 0
|
||||
if current:
|
||||
yield current
|
||||
|
||||
|
||||
class Filesystems:
|
||||
"""Hold filesystem error cursors from before the first destination write."""
|
||||
|
||||
def __init__(self, paths: Sequence[Path], deadline: float):
|
||||
self.handles: dict[int, tuple[Path, int, int]] = {}
|
||||
self.closed = False
|
||||
remaining(deadline)
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
try:
|
||||
self._syncfs = libc.syncfs
|
||||
except AttributeError as error:
|
||||
raise DeliveryError("recovery filesystem synchronization unavailable") from error
|
||||
self._syncfs.argtypes, self._syncfs.restype = [ctypes.c_int], ctypes.c_int
|
||||
try:
|
||||
for path in paths:
|
||||
remaining(deadline)
|
||||
while not path.exists() and not path.is_symlink():
|
||||
if path.parent == path:
|
||||
raise DeliveryError("recovery filesystem ancestor unavailable")
|
||||
path = path.parent
|
||||
before = path.lstat()
|
||||
if not stat.S_ISDIR(before.st_mode):
|
||||
raise DeliveryError("recovery filesystem anchor differs")
|
||||
if before.st_dev in self.handles:
|
||||
continue
|
||||
descriptor = os.open(
|
||||
path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
|
||||
)
|
||||
try:
|
||||
if file_copy._identity(os.fstat(descriptor)) != file_copy._identity(before):
|
||||
raise DeliveryError("recovery filesystem anchor changed")
|
||||
except BaseException:
|
||||
os.close(descriptor)
|
||||
raise
|
||||
self.handles[before.st_dev] = (path, descriptor, before.st_ino)
|
||||
except BaseException:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def contains(self, path: Path) -> None:
|
||||
parent = path.parent.lstat()
|
||||
if self.closed or not stat.S_ISDIR(parent.st_mode) or parent.st_dev not in self.handles:
|
||||
raise DeliveryError("recovery destination has no pre-write filesystem cursor")
|
||||
|
||||
def sync(self, deadline: float) -> None:
|
||||
if self.closed:
|
||||
raise DeliveryError("recovery filesystem cursor is closed")
|
||||
for device, (path, descriptor, inode) in self.handles.items():
|
||||
remaining(deadline)
|
||||
current, opened = path.lstat(), os.fstat(descriptor)
|
||||
# This operator legitimately changes directory entries and modes.
|
||||
# The retained descriptor must still refer to the same directory.
|
||||
if any(
|
||||
not stat.S_ISDIR(info.st_mode) or (info.st_dev, info.st_ino) != (device, inode)
|
||||
for info in (current, opened)
|
||||
):
|
||||
raise DeliveryError("recovery filesystem anchor changed")
|
||||
if self._syncfs(descriptor) != 0:
|
||||
raise DeliveryError("recovery filesystem synchronization failed")
|
||||
remaining(deadline)
|
||||
|
||||
def close(self) -> None:
|
||||
if not self.closed:
|
||||
self.closed = True
|
||||
error = None
|
||||
for _, descriptor, _ in self.handles.values():
|
||||
try:
|
||||
os.close(descriptor)
|
||||
except OSError as failure:
|
||||
error = failure
|
||||
if error is not None:
|
||||
raise error
|
||||
|
||||
|
||||
def pending_file(
|
||||
source: Path, path: Path, item: dict, uid: int, gid: int, mode: int, deadline: float
|
||||
) -> tuple[int, ...]:
|
||||
"""Materialize only; this witness cannot certify bytes before batch readback."""
|
||||
remaining(deadline)
|
||||
if path.exists() or path.is_symlink():
|
||||
info = path.lstat()
|
||||
if (
|
||||
not stat.S_ISREG(info.st_mode)
|
||||
or info.st_nlink != 1
|
||||
or (info.st_uid, info.st_gid, stat.S_IMODE(info.st_mode), info.st_size)
|
||||
!= (uid, gid, mode, item["bytes"])
|
||||
):
|
||||
raise DeliveryError("existing recovery destination differs; no overwrite")
|
||||
return file_copy._identity(info)
|
||||
with (
|
||||
os.fdopen(os.open(source, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC), "rb") as incoming,
|
||||
os.fdopen(
|
||||
os.open(
|
||||
path,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
0o600,
|
||||
),
|
||||
"wb",
|
||||
) as outgoing,
|
||||
):
|
||||
file_copy.transfer(incoming, outgoing, item["bytes"], deadline=deadline)
|
||||
outgoing.flush()
|
||||
os.fchown(outgoing.fileno(), uid, gid)
|
||||
os.fchmod(outgoing.fileno(), mode)
|
||||
witness = file_copy._identity(os.fstat(outgoing.fileno()))
|
||||
if file_copy._identity(path.lstat()) != witness:
|
||||
raise DeliveryError("recovery destination changed before synchronization")
|
||||
return witness
|
||||
|
||||
|
||||
def materialize(
|
||||
rows: Sequence[dict],
|
||||
source: Callable[[dict], Path],
|
||||
destination: Callable[[dict], tuple[Path, int, int, int]],
|
||||
readback: Callable[[Path, dict, int, int, int, float], tuple[int, ...]],
|
||||
filesystems: Filesystems,
|
||||
deadline: float,
|
||||
) -> dict[str, tuple[int, ...]]:
|
||||
"""Copy, join, sync and independently read each complete destination.
|
||||
|
||||
Only four calls are dispatched at a time. A batch retains at most 256 file
|
||||
records and 64 MiB of logical contents; a larger file is handled alone.
|
||||
Every returned witness follows an actual filesystem flush and full SHA256
|
||||
destination read. Exceptions join outstanding work and retain partial files.
|
||||
"""
|
||||
remaining(deadline)
|
||||
verified = {}
|
||||
with ThreadPoolExecutor(max_workers=file_copy.IN_FLIGHT) as executor:
|
||||
for batch in batches(rows):
|
||||
pending = []
|
||||
for offset in range(0, len(batch), file_copy.IN_FLIGHT):
|
||||
remaining(deadline)
|
||||
group = [
|
||||
(item, *destination(item))
|
||||
for item in batch[offset : offset + file_copy.IN_FLIGHT]
|
||||
]
|
||||
for _, path, _, _, _ in group:
|
||||
filesystems.contains(path)
|
||||
calls = [
|
||||
executor.submit(
|
||||
pending_file, source(item), path, item, uid, gid, mode, deadline
|
||||
)
|
||||
for item, path, uid, gid, mode in group
|
||||
]
|
||||
for record, call in zip(group, calls, strict=True):
|
||||
pending.append((*record, call.result()))
|
||||
filesystems.sync(deadline)
|
||||
for offset in range(0, len(pending), file_copy.IN_FLIGHT):
|
||||
remaining(deadline)
|
||||
group = pending[offset : offset + file_copy.IN_FLIGHT]
|
||||
calls = [
|
||||
executor.submit(readback, path, item, uid, gid, mode, deadline)
|
||||
for item, path, uid, gid, mode, _ in group
|
||||
]
|
||||
for (item, _, _, _, _, before), call in zip(group, calls, strict=True):
|
||||
actual = call.result()
|
||||
if actual != before:
|
||||
raise DeliveryError("recovery destination changed across synchronization")
|
||||
verified[item["path"]] = actual
|
||||
remaining(deadline)
|
||||
return verified
|
||||
Reference in New Issue
Block a user