perf(delivery): reduce verified installation and recovery I/O

scripts/release/README.md

@@ -398,6 +398,16 @@ digest. Its inode, ownership, mode, link count, size and nanosecond timestamps
 must remain unchanged until the final receipt. These observations are local to
 one stopped-node recovery and are never reused after an interrupted attempt.
 Directory entries are flushed after their children before completion is recorded.
+The cold operator, activation and immutable service/dependency/journal profile
+installation can reuse a public bundle's complete byte verification within one
+synchronous operation. Every file and directory identity, owner, mode, link count,
+size and nanosecond mtime/ctime is checked before reuse and before success returns.
+Changes are refused. These memory-only proofs are bound to the process, capped at
+four bundles, 100006 entries and 16 MiB of metadata, and discarded on return or
+failure. Further bundles receive full verification; separate commands start fresh.
+Fixed-size regular-file copies can use Linux kernel transfer with bounded chunks.
+Unsupported transfer falls back at the same offsets. Source size/EOF, file fsync
+and independent destination byte verification remain required.
 Old configuration and maintenance receipts go into inactive
 `recovery/ID/history/`. The new boot's Sandbox preparation owns cgroup recreation.
 Unknown/nonempty targets and differing existing files are refused; partial restores

scripts/release/dependency_install.py

@@ -15,7 +15,7 @@ import subprocess
 from pathlib import Path
 from urllib.parse import urlsplit
 
-from . import configuration, dependencies, install, preflight, service_install
+from . import configuration, dependencies, install, manifest, preflight, service_install
 from .policy import DeliveryError, canonical, digest, fields, read_json, sha256
 
 ROOT = dependencies.CONFIGURATION_ROOT
@@ -387,6 +387,7 @@ def install_cookie(source: Path, expected: str) -> None:
     install.sync_directory(path.parent)
 
 
+@manifest.verified_operation
 def install_snapshot(source: Path, expected: str, cookie: Path, cookie_sha: str) -> dict:
     if os.getuid() != 0 or os.geteuid() != 0:
         raise DeliveryError("dependency installation requires root operator")
@@ -405,7 +406,13 @@ def install_snapshot(source: Path, expected: str, cookie: Path, cookie_sha: str)
     ):
         raise DeliveryError("dependency snapshot is not the selected binding")
     payload = install.RELEASES / selected["release_id"] / "payload"
-    for name in ("dependencies.py", "dependency_policy.py", "dependency_install.py", "layout.py"):
+    for name in (
+        "dependencies.py",
+        "dependency_policy.py",
+        "dependency_install.py",
+        "layout.py",
+        "manifest.py",
+    ):
         shipped = payload / "delivery/scripts/release" / name
         preflight.protected(shipped)
         if digest(shipped) != digest(Path(__file__).parent / name):

scripts/release/file_copy.py

@@ -0,0 +1,81 @@
+"""Fixed-size regular-file transfer; callers retain durability and byte readback."""
+
+import errno
+import os
+import stat
+import time
+from typing import BinaryIO
+
+from .policy import MAX_FILE_BYTES, DeliveryError
+
+CHUNK = 1024 * 1024
+UNSUPPORTED = {errno.ENOSYS, errno.EINVAL, errno.EOPNOTSUPP, errno.EXDEV}
+
+
+def _identity(info: os.stat_result) -> tuple[int, ...]:
+    return (
+        info.st_dev,
+        info.st_ino,
+        info.st_mode,
+        info.st_nlink,
+        info.st_uid,
+        info.st_gid,
+        info.st_size,
+        info.st_mtime_ns,
+        info.st_ctime_ns,
+    )
+
+
+def transfer(
+    incoming: BinaryIO, outgoing: BinaryIO, expected_bytes: int, *, deadline: float | None = None
+) -> None:
+    """Use fresh streams; unsupported kernel transfer resumes at the same offsets."""
+    source, target = incoming.fileno(), outgoing.fileno()
+    before = os.fstat(source)
+    destination = os.fstat(target)
+    if (
+        type(expected_bytes) is not int
+        or not 0 <= expected_bytes <= MAX_FILE_BYTES
+        or not stat.S_ISREG(before.st_mode)
+        or before.st_nlink != 1
+        or before.st_size != expected_bytes
+        or not stat.S_ISREG(destination.st_mode)
+        or destination.st_nlink != 1
+        or destination.st_size != 0
+        or incoming.tell() != 0
+        or outgoing.tell() != 0
+        or os.lseek(source, 0, os.SEEK_CUR) != 0
+        or os.lseek(target, 0, os.SEEK_CUR) != 0
+    ):
+        raise DeliveryError("fixed-size regular copy input or fresh destination differs")
+
+    def remaining() -> None:
+        if deadline is not None and time.monotonic() >= deadline:
+            raise DeliveryError("copy deadline exceeded; private partial target retained")
+
+    total = 0
+    kernel = hasattr(os, "sendfile")
+    while total < expected_bytes:
+        remaining()
+        maximum = min(CHUNK, expected_bytes - total)
+        if kernel:
+            try:
+                # Linux None advances the source offset, including a copied prefix
+                # before falling back. Each call stays within the verified size.
+                count = os.sendfile(target, source, None, maximum)
+            except OSError as error:
+                if error.errno not in UNSUPPORTED:
+                    raise
+                kernel = False
+                continue
+        else:
+            data = incoming.read(maximum)
+            count = len(data)
+            if count and outgoing.write(data) != count:
+                raise DeliveryError("short copy write; private partial target retained")
+        if not 0 < count <= maximum:
+            raise DeliveryError("copy input bytes differ; private partial target retained")
+        total += count
+    remaining()
+    if incoming.read(1) or _identity(os.fstat(source)) != _identity(before):
+        raise DeliveryError("copy input changed; private partial target retained")

scripts/release/install.py

@@ -8,7 +8,7 @@ import stat
 import subprocess
 from pathlib import Path
 
-from . import manifest, preflight
+from . import file_copy, manifest, preflight
 from .layout import RELEASES as RELEASES
 from .policy import DeliveryError, canonical, digest, fields, read_json, sha256
 
@@ -93,7 +93,15 @@ def copy_verified(source: Path, target: Path, expected: str, mode: int) -> None:
     _copy_verified(source, target, expected, mode, readback=True)
 
 
-def _copy_verified(source: Path, target: Path, expected: str, mode: int, *, readback: bool) -> None:
+def _copy_verified(
+    source: Path,
+    target: Path,
+    expected: str,
+    mode: int,
+    *,
+    readback: bool,
+    expected_bytes: int | None = None,
+) -> None:
     preflight.protected(source)
     sha256(expected)
     if target.exists() or target.is_symlink():
@@ -116,10 +124,13 @@ def _copy_verified(source: Path, target: Path, expected: str, mode: int, *, read
             # below verifies every copied byte against the trusted records;
             # hashing the same stream as well adds no independent evidence.
             copied = hashlib.sha256() if readback else None
-            while chunk := src.read(1024 * 1024):
-                if copied is not None:
+            if copied is None:
+                size = os.fstat(incoming).st_size if expected_bytes is None else expected_bytes
+                file_copy.transfer(src, dst, size)
+            else:
+                while chunk := src.read(1024 * 1024):
                     copied.update(chunk)
-                dst.write(chunk)
+                    dst.write(chunk)
             dst.flush()
             if copied is not None and copied.hexdigest() != expected:
                 raise DeliveryError(
@@ -233,6 +244,7 @@ def frozen_copy(
             record["sha256"],
             int(record["mode"], 8),
             readback=False,
+            expected_bytes=record["bytes"],
         )
     for name in sorted(directories, key=lambda name: name.count("/"), reverse=True):
         (target / name).chmod(0o555)
@@ -454,6 +466,7 @@ def require_stopped() -> None:
             raise DeliveryError("delegated task groups remain; cleanup proof required")
 
 
+@manifest.verified_operation
 def activate(release_id: str, expected: str, batch: str) -> dict:
     """Only files + separately installed exact configuration may be selected.
 

scripts/release/journal.py

@@ -9,7 +9,7 @@ import stat
 import subprocess
 from pathlib import Path
 
-from . import configuration, install, preflight
+from . import configuration, install, manifest, preflight
 from .policy import DeliveryError, canonical, digest, fields, read_json
 
 NAMESPACE = "kf-onlinejudge"
@@ -286,15 +286,17 @@ def check_ready() -> dict:
     return value
 
 
+@manifest.verified_operation
 def install_profile() -> dict:
     if os.getuid() != 0 or os.geteuid() != 0 or os.getgid() != 0:
         raise DeliveryError("root journal operator required")
     selected = install.selection()
     payload = install.RELEASES / selected["release_id"] / "payload"
-    shipped = payload / "delivery/scripts/release/journal.py"
-    preflight.protected(shipped)
-    if digest(shipped) != digest(Path(__file__)):
-        raise DeliveryError("journal operator differs from selected delivery")
+    for name in ("journal.py", "manifest.py"):
+        shipped = payload / "delivery/scripts/release" / name
+        preflight.protected(shipped)
+        if digest(shipped) != digest(Path(__file__).parent / name):
+            raise DeliveryError("journal operator differs from selected delivery")
     install.require_stopped()
     observed = storage()
     if RECEIPT.exists() or RECEIPT.is_symlink():

scripts/release/manifest.py

@@ -1,8 +1,13 @@
 """Explicit directory bundle inventories, verified before installation."""
 
+import json
 import os
 import re
 import stat
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass, field
+from functools import wraps
 from pathlib import Path
 
 from .policy import (
@@ -24,6 +29,104 @@ from .policy import (
 )
 
 
+@dataclass(slots=True)
+class _ReadScope:
+    pid: int
+    entries: dict = field(default_factory=dict)
+    nodes: int = 0
+    metadata_bytes: int = 0
+
+
+_READ_SCOPE: ContextVar[_ReadScope | None] = ContextVar("delivery_bundle_reads", default=None)
+_MAX_SCOPED_BUNDLES = 4
+_MAX_SCOPED_NODES = 2 * (MAX_FILES + 3)
+_MAX_SCOPED_METADATA = 2 * MAX_METADATA_BYTES
+
+
+def _bundle_signature(bundle: Path) -> dict[str, tuple[int, ...]]:
+    """Observe topology and all metadata that can invalidate a prior byte read."""
+    if not bundle.is_dir() or bundle.is_symlink():
+        raise DeliveryError("bundle directory required")
+    members = set()
+    for path in bundle.iterdir():
+        members.add(path.name)
+        if len(members) > 2:
+            raise DeliveryError("unexpected bundle member")
+    if members != {"manifest.json", "payload"}:
+        raise DeliveryError("unexpected bundle member")
+    result = {}
+
+    def add(path: Path, *, directory: bool) -> None:
+        info = path.lstat()
+        if not (stat.S_ISDIR(info.st_mode) if directory else stat.S_ISREG(info.st_mode)):
+            raise DeliveryError("bundle proof requires ordinary directories and files")
+        if not directory and info.st_nlink != 1:
+            raise DeliveryError("bundle proof requires single-link files")
+        result[path.relative_to(bundle).as_posix()] = (
+            info.st_dev,
+            info.st_ino,
+            info.st_mode,
+            info.st_nlink,
+            info.st_uid,
+            info.st_gid,
+            info.st_size,
+            info.st_mtime_ns,
+            info.st_ctime_ns,
+        )
+        if len(result) > MAX_FILES + 3:
+            raise DeliveryError("bundle proof entry count limit")
+
+    def failed(error: OSError) -> None:
+        raise error
+
+    add(bundle, directory=True)
+    add(bundle / "manifest.json", directory=False)
+    payload = bundle / "payload"
+    add(payload, directory=True)
+    for directory, names, files in os.walk(payload, followlinks=False, onerror=failed):
+        for name in names:
+            path = Path(directory) / name
+            relative_name(path.relative_to(payload).as_posix() + "/.__directory__")
+            add(path, directory=True)
+        for name in files:
+            path = Path(directory) / name
+            relative_name(path.relative_to(payload).as_posix())
+            add(path, directory=False)
+    return result
+
+
+def _unchanged_bundle(bundle: Path, signature: dict[str, tuple[int, ...]]) -> None:
+    if _bundle_signature(bundle) != signature:
+        raise DeliveryError("verified bundle changed during the operator operation")
+
+
+@contextmanager
+def verification_scope():
+    """Reuse actual byte reads only inside one synchronous same-PID operation."""
+    previous = _READ_SCOPE.get()
+    if previous is not None and previous.pid == os.getpid():
+        yield
+        return
+    current = _ReadScope(os.getpid())
+    token = _READ_SCOPE.set(current)
+    try:
+        yield
+        # This happens before the decorated operation returns its success value.
+        for (bundle, _), (_, signature) in current.entries.items():
+            _unchanged_bundle(bundle, signature)
+    finally:
+        _READ_SCOPE.reset(token)
+
+
+def verified_operation(function):
+    @wraps(function)
+    def wrapped(*args, **kwargs):
+        with verification_scope():
+            return function(*args, **kwargs)
+
+    return wrapped
+
+
 def inventory(root: Path, *, path_prefix: str = "") -> list[dict]:
     """No links/devices or inferred inclusion; bounds apply before hashing."""
     if not root.is_dir() or root.is_symlink():
@@ -203,6 +306,34 @@ def validate(data: dict) -> dict:
 
 
 def verify(bundle: Path, expected_sha256: str) -> dict:
+    """A bounded operation may reuse a complete byte proof only while unchanged."""
+    expected_sha256 = sha256(expected_sha256)
+    scope = _READ_SCOPE.get()
+    if scope is None or scope.pid != os.getpid():
+        return _verify_contents(bundle, expected_sha256)
+    key = (bundle, expected_sha256)
+    if key in scope.entries:
+        data, signature = scope.entries[key]
+        _unchanged_bundle(bundle, signature)
+        # Never expose the remembered metadata object to a mutating caller.
+        return json.loads(data)
+    if len(scope.entries) >= _MAX_SCOPED_BUNDLES:
+        return _verify_contents(bundle, expected_sha256)
+    before = _bundle_signature(bundle)
+    data = _verify_contents(bundle, expected_sha256)
+    _unchanged_bundle(bundle, before)
+    encoded = canonical(data)
+    if (
+        scope.nodes + len(before) <= _MAX_SCOPED_NODES
+        and scope.metadata_bytes + len(encoded) <= _MAX_SCOPED_METADATA
+    ):
+        scope.entries[key] = encoded, before
+        scope.nodes += len(before)
+        scope.metadata_bytes += len(encoded)
+    return data
+
+
+def _verify_contents(bundle: Path, expected_sha256: str) -> dict:
     """The manifest digest comes from a separate trusted operator receipt."""
     expected_sha256 = sha256(expected_sha256)
     if not bundle.is_dir() or bundle.is_symlink():

scripts/release/restore.py

@@ -17,6 +17,7 @@ from . import (
     backup_manifest,
     dependencies,
     dependency_install,
+    file_copy,
     install,
     lifecycle,
     manifest,
@@ -186,16 +187,7 @@ def write_file(
             ),
             "wb",
         ) as outgoing:
-            total = 0
-            while block := incoming.read(1024 * 1024):
-                total += len(block)
-                if total > item["bytes"] or time.monotonic() >= deadline:
-                    raise DeliveryError(
-                        "recovery input/deadline changed; private partial target retained"
-                    )
-                outgoing.write(block)
-            if total != item["bytes"]:
-                raise DeliveryError("recovery input bytes differ; private partial target retained")
+            file_copy.transfer(incoming, outgoing, item["bytes"], deadline=deadline)
             outgoing.flush()
             os.fchown(outgoing.fileno(), uid, gid)
             os.fchmod(outgoing.fileno(), mode)
@@ -206,12 +198,20 @@ def write_file(
     return readback(path, item, uid, gid, mode, deadline)
 
 
+@manifest.verified_operation
 def restore(recovery_id: str, backup_id: str, expected: str) -> dict:
     guarded()
     stopped()
     selected = install.selection()
     payload = install.RELEASES / selected["release_id"] / "payload"
-    for name in ("restore.py", "restore_policy.py", "backup.py", "backup_manifest.py"):
+    for name in (
+        "restore.py",
+        "restore_policy.py",
+        "backup.py",
+        "backup_manifest.py",
+        "manifest.py",
+        "file_copy.py",
+    ):
         if digest(payload / "delivery/scripts/release" / name) != digest(
             Path(__file__).parent / name
         ):
@@ -300,8 +300,11 @@ def restore(recovery_id: str, backup_id: str, expected: str) -> dict:
             value["directories"], key=lambda row: row["path"].count("/"), reverse=True
         ):
             path, uid, gid, mode = target(item, recovery)
-            os.chown(path, uid, gid, follow_symlinks=False)
-            path.chmod(mode)
+            current = path.lstat()
+            if (current.st_uid, current.st_gid) != (uid, gid):
+                os.chown(path, uid, gid, follow_symlinks=False)
+            if stat.S_IMODE(current.st_mode) != mode:
+                path.chmod(mode)
             install.sync_directory(path)
             member(path, uid, gid, mode, directory=True)
         # Flush directory entries once after their children, including the two

scripts/release/service_install.py

@@ -10,7 +10,7 @@ import stat
 import subprocess
 from pathlib import Path
 
-from . import configuration, install, preflight, services
+from . import configuration, install, manifest, preflight, services
 from .policy import DeliveryError, canonical, digest, fields, read_json, sha256
 
 UNITS = Path("/etc/kf-onlinejudge/units")
@@ -155,6 +155,7 @@ def systemctl(*arguments: str) -> str:
     return result.stdout
 
 
+@manifest.verified_operation
 def install_units(source: Path, expected: str) -> dict:
     if os.getuid() != 0 or os.geteuid() != 0:
         raise DeliveryError("ordinary unit installation requires root operator")
@@ -184,6 +185,7 @@ def install_units(source: Path, expected: str) -> dict:
         "service_bootstrap.py",
         "service_install.py",
         "layout.py",
+        "manifest.py",
     ):
         shipped = payload / "delivery/scripts/release" / name
         preflight.protected(shipped)