perf(delivery): use bounded native cold restore copies

2026-10-28 12:11:53 +08:00
parent 9f4a8c9a68
commit 8d7f651ddc
24 changed files with 2911 additions and 19 deletions
@@ -20,6 +20,7 @@ from kf_oj.contracts.problem_maintenance import (
    MaintenanceReceipt,
    PublishCandidate,
 )
+from kf_oj.contracts.restore_io import RestoreIOReply, RestoreIORequest
 from kf_oj.contracts.submission_performance import SubmissionPerformance
 from kf_oj.core.settings import Settings

@@ -57,6 +58,8 @@ def main() -> None:
    outputs[root / "contracts/schemas/submission-performance.v1.schema.json"] = performance
    for name, model in {
        "delivery-availability": AvailabilityRecord,
+        "restore-io-request": RestoreIORequest,
+        "restore-io-reply": RestoreIOReply,
        "problem-publish": PublishCandidate,
        "problem-availability": ChangeProblemAvailability,
        "problem-maintenance": MaintenanceReceipt,
@@ -0,0 +1,129 @@
+"""Private offline restore byte-copy protocol; conveys no execution authority."""
+
+import re
+from typing import Annotated, Literal
+
+from pydantic import (
+    BaseModel,
+    BeforeValidator,
+    ConfigDict,
+    Field,
+    RootModel,
+    StrictInt,
+    model_validator,
+)
+
+from kf_oj.contracts.judging import integer_version
+
+MAX_FILES = 98304
+MAX_LOGICAL_BYTES = 12 * 1024**3
+MAX_FILE_BYTES = 256 * 1024**2
+MAX_BATCH_BYTES = 64 * 1024**2
+DOMAINS = frozenset(
+    "postgres artifacts imports app-config judge-config factory operations releases "
+    "images dependencies proxy recovery".split()
+)
+Count = Annotated[StrictInt, Field(ge=0, le=MAX_FILES)]
+ByteCount = Annotated[StrictInt, Field(ge=0, le=MAX_LOGICAL_BYTES)]
+
+
+class Record(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
+
+
+class RestoreFile(Record):
+    path: Annotated[str, Field(min_length=1, max_length=512)]
+    uid: Annotated[StrictInt, Field(ge=0, le=1205)]
+    gid: Annotated[StrictInt, Field(ge=0, le=1205)]
+    mode: StrictInt
+    bytes: Annotated[StrictInt, Field(ge=0, le=MAX_FILE_BYTES)]
+    sha256: Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]
+
+    @model_validator(mode="after")
+    def check_member(self) -> "RestoreFile":
+        parts = self.path.split("/")
+        if (
+            len(parts) < 2
+            or parts[0] not in DOMAINS
+            or any(
+                p in {"", ".", "..", ".git", ".ssh", ".localplan", "docs"}
+                or re.fullmatch(r"[A-Za-z0-9_.+@-]{1,255}", p) is None
+                for p in parts
+            )
+        ):
+            raise ValueError("restore file must be below a fixed backup domain")
+        if self.uid not in {0, 1200, 1201, 1202, 1203, 1204, 1205} or self.gid not in (
+            {0, 1203, 1204, 1205} if self.uid == 0 else {self.uid}
+        ):
+            raise ValueError("unsupported restore identity")
+        if self.mode not in {0o400, 0o444, 0o600, 0o640, 0o644, 0o555, 0o755}:
+            raise ValueError("unsupported restore file mode")
+        if self.sha256 in {"a" * 64, "0" * 64}:
+            raise ValueError("placeholder digest refused")
+        return self
+
+
+class Open(Record):
+    schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
+    kind: Literal["restore.io.open"]
+    backup_id: Annotated[str, Field(pattern=r"^bk-[0-9a-f]{32}$")]
+    recovery_id: Annotated[str, Field(pattern=r"^rr-[0-9a-f]{32}$")]
+    expected_files: Annotated[StrictInt, Field(ge=1, le=MAX_FILES)]
+    expected_logical_bytes: ByteCount
+    deadline_monotonic_ns: Annotated[StrictInt, Field(ge=1, le=2**63 - 1)]
+
+
+class Batch(Record):
+    schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
+    kind: Literal["restore.io.batch"]
+    batch_id: Annotated[StrictInt, Field(ge=1, le=MAX_FILES)]
+    files: Annotated[list[RestoreFile], Field(min_length=1, max_length=256)]
+
+    @model_validator(mode="after")
+    def check_batch(self) -> "Batch":
+        if len(self.files) > 1 and sum(f.bytes for f in self.files) > MAX_BATCH_BYTES:
+            raise ValueError("restore batch byte bound exceeded")
+        if len({f.path for f in self.files}) != len(self.files):
+            raise ValueError("duplicate restore batch member")
+        return self
+
+
+class Finish(Record):
+    schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
+    kind: Literal["restore.io.finish"]
+
+
+class RestoreIORequest(RootModel[Annotated[Open | Batch | Finish, Field(discriminator="kind")]]):
+    pass
+
+
+class RestoreIOReply(Record):
+    schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
+    kind: Literal["restore.io.reply"]
+    stage: Literal["opened", "verified", "finished", "refused"]
+    batch_id: Count
+    verified_files: Count
+    verified_logical_bytes: ByteCount
+    execution_verified: Annotated[
+        Literal[False],
+        BeforeValidator(lambda value: value if type(value) is bool else _bad_boolean()),
+    ]
+    error: Literal["none", "protocol", "guard", "deadline", "filesystem", "bytes", "identity"]
+
+    @model_validator(mode="after")
+    def check_reply(self) -> "RestoreIOReply":
+        if (self.stage == "refused") != (self.error != "none"):
+            raise ValueError("restore refusal state differs")
+        if self.stage == "opened" and (
+            self.batch_id or self.verified_files or self.verified_logical_bytes
+        ):
+            raise ValueError("restore open cannot claim verified data")
+        if self.stage in {"verified", "finished"} and not (
+            1 <= self.batch_id <= self.verified_files
+        ):
+            raise ValueError("restore completion has no verified batch")
+        return self
+
+
+def _bad_boolean() -> bool:
+    raise ValueError("execution verification must be a boolean false")
@@ -1437,3 +1437,35 @@ def test_proxy_previous_snapshot_retains_the_legacy_unit_and_binding_format(tmp_
    unit.chmod(0o644)
    assert proxy_install.known_snapshot(directory) == selected
    assert "--previous-certificate" not in unit.read_text()
+
+
+@pytest.mark.parametrize("support", ["legacy", "complete", "binary-only", "transport-only"])
+def test_native_restore_capability_requires_a_complete_pair_but_keeps_legacy(
+    ordinary_bundle, support
+):
+    bundle, _ = ordinary_bundle
+    data = json.loads((bundle / "manifest.json").read_bytes())
+    names = {
+        "legacy": [],
+        "complete": ["bin/kf-restore-io", "delivery/scripts/release/restore_native.py"],
+        "binary-only": ["bin/kf-restore-io"],
+        "transport-only": ["delivery/scripts/release/restore_native.py"],
+    }[support]
+    parents = set(data["directories"])
+    for name in names:
+        data["files"].append(
+            {
+                "path": name,
+                "bytes": 1,
+                "sha256": "b" * 64,
+                "mode": "0555" if name.startswith("bin/") else "0444",
+            }
+        )
+        data["payload_bytes"] += 1
+        parents.update(str(p) for p in Path(name).parents if str(p) != ".")
+    data["directories"] = sorted(parents)
+    if support in {"legacy", "complete"}:
+        assert manifest.validate(data) is data
+    else:
+        with pytest.raises(policy.DeliveryError, match="incomplete application bundle"):
+            manifest.validate(data)
@@ -0,0 +1,23 @@
+"""Shared ordinary wire examples; no privileged copy or execution acceptance."""
+
+import json
+from pathlib import Path
+
+import pytest
+from pydantic import ValidationError
+
+from kf_oj.contracts.restore_io import RestoreIOReply, RestoreIORequest
+
+CASES = json.loads(
+    (Path(__file__).resolve().parents[2] / "tests/fixtures/restore-io/examples.json").read_bytes()
+)
+
+
+@pytest.mark.parametrize("case", CASES, ids=lambda c: c["name"])
+def test_shared_restore_io_examples(case):
+    model = RestoreIORequest if case["direction"] == "request" else RestoreIOReply
+    if case["valid"]:
+        assert model.model_validate(case["value"]).model_dump(mode="json") == case["value"]
+    else:
+        with pytest.raises(ValidationError):
+            model.model_validate(case["value"])
@@ -0,0 +1,115 @@
+"""Ordinary pipe fixtures only; no Root, filesystem-copy or VM acceptance."""
+
+import json
+import subprocess
+import sys
+import time
+
+import pytest
+from scripts.release import restore_native
+from scripts.release.policy import DeliveryError
+
+
+@pytest.fixture
+def transport(tmp_path, monkeypatch):
+    helper = tmp_path / "bin/kf-restore-io"
+    helper.parent.mkdir()
+    helper.write_bytes(b"ordinary metadata fixture, never executed\n")
+    helper.chmod(0o555)
+    monkeypatch.setattr(restore_native.preflight, "protected", lambda *_a, **_kw: None)
+    original = subprocess.Popen
+    children = []
+
+    def start(program):
+        def popen(args, **options):
+            assert args == [str(helper), "--stdio-v1"]
+            assert options["start_new_session"] and options["close_fds"]
+            assert set(options["env"]) == {"PATH", "LC_ALL", "LD_LIBRARY_PATH"}
+            child = original([sys.executable, "-c", program], **options)
+            children.append(child)
+            return child
+
+        monkeypatch.setattr(restore_native.subprocess, "Popen", popen)
+
+    yield tmp_path, start, children
+    assert all(p.poll() is not None for p in children)
+    assert all(h.closed for p in children for h in (p.stdin, p.stdout, p.stderr))
+
+
+def test_transport_counts_and_child_join_without_execution_claim(transport):
+    payload, start, children = transport
+    start("""
+import json,sys
+count=total=batch=0
+for line in sys.stdin:
+ value=json.loads(line)
+ kind=value['kind']
+ stage='opened'
+ if kind=='restore.io.batch':
+  count+=len(value['files'])
+  total+=sum(r['bytes'] for r in value['files'])
+  batch=value['batch_id']
+  stage='verified'
+ if kind=='restore.io.finish':stage='finished'
+ reply={'schema_version':1,'kind':'restore.io.reply','stage':stage,'batch_id':batch,'verified_files':count,'verified_logical_bytes':total,'execution_verified':False,'error':'none'}
+ print(json.dumps(reply),flush=True)
+ if stage=='finished':break
+""")
+    session = restore_native.Session(
+        payload, "bk-" + "1" * 32, "rr-" + "2" * 32, 2, 7, time.monotonic() + 10
+    )
+    try:
+        session.materialize(
+            [
+                {"path": "artifacts/a", "bytes": 3, "sha256": "1" * 64},
+                {"path": "artifacts/b", "bytes": 4, "sha256": "2" * 64},
+            ]
+        )
+        session.finish()
+        assert session.finished and children[0].returncode == 0
+    finally:
+        session.close()
+
+
+@pytest.mark.parametrize(
+    "response",
+    [
+        b"",
+        b"not-json\n",
+        b"{}\n{}\n",
+        b"x" * (64 * 1024 + 1),
+        json.dumps(
+            {
+                "schema_version": 1,
+                "kind": "restore.io.reply",
+                "stage": "opened",
+                "batch_id": 0,
+                "verified_files": 0,
+                "verified_logical_bytes": 0,
+                "execution_verified": True,
+                "error": "none",
+            }
+        ).encode()
+        + b"\n",
+    ],
+)
+def test_transport_refuses_eof_malformed_unbounded_or_authority_reply(transport, response):
+    payload, start, _ = transport
+    start(
+        "import sys;sys.stdin.readline();sys.stdout.buffer.write("
+        + repr(response)
+        + ");sys.stdout.flush()"
+    )
+    with pytest.raises(DeliveryError):
+        restore_native.Session(
+            payload, "bk-" + "1" * 32, "rr-" + "2" * 32, 1, 0, time.monotonic() + 10
+        )
+
+
+def test_transport_deadline_reaps_its_owned_waiting_child(transport):
+    payload, start, _ = transport
+    start("import sys,time;sys.stdin.readline();time.sleep(10)")
+    then = time.monotonic()
+    with pytest.raises(DeliveryError):
+        restore_native.Session(payload, "bk-" + "1" * 32, "rr-" + "2" * 32, 1, 0, then + 0.2)
+    assert time.monotonic() - then < 5