Kronecker/KF-OnlineJudge
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(delivery): avoid duplicate installed byte reads

Browse Source
This commit is contained in:
Kronecker
2026-10-21 14:41:56 +08:00
parent d66d42d021
commit 7650a7e7b6
3 changed files with 161 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_delivery_io.py
+133
View File
@@ -23,6 +23,7 @@ factory_service = importlib.import_module("scripts.release.factory_service")
proxy_install = importlib.import_module("scripts.release.proxy_install")
setup_session = importlib.import_module("scripts.release.setup_session")
lifecycle = importlib.import_module("scripts.release.lifecycle")
installer = importlib.import_module("scripts.release.install")
@pytest.fixture
@@ -103,6 +104,138 @@ def observed_digests(monkeypatch):
return reads
@pytest.fixture
def ordinary_installation_model(ordinary_bundle, monkeypatch, tmp_path):
"""Owned dummy bytes; Root accounts, kernel and ownership gates are models."""
bundle, _ = ordinary_bundle
data = json.loads((bundle / "manifest.json").read_bytes())
for name, contents in (
("deploy/release/kfonlinejudge.slice", b"ordinary unit fixture, never started\n"),
("images/gnu/bin/ordinary-data", b"ordinary image bytes, never executed\n"),
):
path = bundle / "payload" / name
path.parent.parent.chmod(0o755)
path.parent.mkdir(mode=0o755, exist_ok=True)
path.parent.chmod(0o755)
path.write_bytes(contents)
path.chmod(0o444)
for path in sorted((bundle / "payload").rglob("*"), key=lambda p: len(p.parts), reverse=True):
if path.is_dir():
path.chmod(0o555)
(bundle / "manifest.json").unlink()
manifest.write(bundle, data["source_sha"], data["locks"], data["images"])
expected = policy.digest(bundle / "manifest.json")
root = tmp_path / "ordinary-installation-model"
(root / "etc/systemd/system").mkdir(parents=True)
def fixture_path(value):
value = Path(value)
if value.is_absolute() and value.parts[1:2] in (("opt",), ("etc",), ("var",)):
return root / value.relative_to("/")
return value
def directory(path, mode):
path.mkdir(mode=mode, parents=True, exist_ok=True)
path.chmod(mode)
monkeypatch.setattr(installer, "Path", fixture_path)
monkeypatch.setattr(installer, "RELEASES", root / "opt/kf-onlinejudge/releases")
monkeypatch.setattr(installer, "IMAGES", root / "var/lib/kf-judge/images")
monkeypatch.setattr(installer.os, "geteuid", lambda: 0)
monkeypatch.setattr(installer, "ensure_accounts", lambda: None)
monkeypatch.setattr(installer.preflight, "protected", lambda *a, **k: None)
monkeypatch.setattr(installer.preflight, "observe", lambda *a, **k: {"eligible": True})
monkeypatch.setattr(installer, "directory", directory)
monkeypatch.setattr(installer, "data_directory", lambda path, owner: directory(path, 0o700))
data = json.loads((bundle / "manifest.json").read_bytes())
return {
"bundle": bundle,
"expected": expected,
"data": data,
"root": root,
"release": installer.RELEASES / data["release_id"],
"stage": installer.RELEASES / ("." + data["release_id"] + ".installing"),
"images": installer.IMAGES / data["release_id"],
}
def test_new_staged_install_reads_each_destination_once(ordinary_installation_model, monkeypatch):
model = ordinary_installation_model
reads = []
original = manifest._read_payload
def read(row):
if model["stage"] in row[0].parents:
reads.append(row[0].relative_to(model["stage"] / "payload").as_posix())
return original(row)
monkeypatch.setattr(manifest, "_read_payload", read)
receipt = installer.files(model["bundle"], model["expected"])
assert Counter(reads) == Counter(row["path"] for row in model["data"]["files"])
assert receipt["files_installed"] and not receipt["activated"]
assert not receipt["migrated"] and not receipt["execution_verified"]
assert model["release"].is_dir() and not model["stage"].exists()
assert policy.digest(model["release"] / "manifest.json") == model["expected"]
def test_staged_install_rechecks_final_payload_before_publication_model(
ordinary_installation_model, monkeypatch
):
model = ordinary_installation_model
original = installer.copy_verified
def copy(source, target, expected, mode):
original(source, target, expected, mode)
if target == model["stage"] / "manifest.json":
# Benign owned fixture bytes only; actual corruption/path safety
# acceptance belongs to the independent VM, not this modeled role.
leaf = model["stage"] / "payload/frontend/index.html"
leaf.chmod(0o600)
leaf.write_bytes(b"different ordinary fixture data\n")
leaf.chmod(0o444)
monkeypatch.setattr(installer, "copy_verified", copy)
with pytest.raises(policy.DeliveryError, match="contents differ"):
installer.files(model["bundle"], model["expected"])
assert not model["release"].exists() and model["stage"].is_dir()
assert not (
model["root"] / "etc/kf-onlinejudge" / (model["data"]["release_id"] + ".files.json")
).exists()
def test_repeat_install_reads_each_GNU_image_once(ordinary_installation_model, monkeypatch):
model = ordinary_installation_model
installer.files(model["bundle"], model["expected"])
root = model["images"] / "gnu"
reads = []
original = manifest._read_payload
def read(row):
if root in row[0].parents:
reads.append(row[0].relative_to(root).as_posix())
return original(row)
monkeypatch.setattr(manifest, "_read_payload", read)
installer.files(model["bundle"], model["expected"])
expected = [
row["path"].removeprefix("images/gnu/")
for row in model["data"]["files"]
if row["path"].startswith("images/gnu/")
]
assert Counter(reads) == Counter(expected)
def test_repeat_install_still_refuses_legacy_byte_change_model(ordinary_installation_model):
model = ordinary_installation_model
installer.files(model["bundle"], model["expected"])
leaf = model["images"] / "legacy-gnu/bin/ordinary-data"
leaf.chmod(0o600)
leaf.write_bytes(b"changed owned ordinary image fixture\n")
leaf.chmod(0o444)
with pytest.raises(policy.DeliveryError, match="legacy compatibility image bytes differ"):
installer.files(model["bundle"], model["expected"])
def test_payload_byte_reads_are_bounded_joined_and_complete(ordinary_bundle, monkeypatch):
# Ordinary owned bytes only. This observes the inventory's actual reads,
# not service permissions, isolation, or VM recovery latency.
scripts/release/README.md
+6
View File
@@ -72,6 +72,12 @@ verified input tree and the fixed target profile. It installs protected versions
images and OS identities. Repeat installation checks identical files; unknown
ownership/modes, changed bytes and incomplete staging directories require inspection.
New release files are copied into a private staging directory, synced and fully
read back with their manifest before atomic publication. Each installed runtime
image has its own full readback. Repeat installation checks the release, runtime
images and legacy GNU copy independently; it uses the already checked GNU records
to compare the legacy copy without reading the runtime image a second time.
`generate-configuration` reads separate 0600 node and endpoint inputs and writes
a new private batch. API, publisher, import, migration and account operations use
their required DB authority; Worker receives its own MQ/control identity. Backend
scripts/release/install.py
+22 -14
View File
@@ -232,13 +232,23 @@ def frozen_copy(
for record in records:
preflight.protected(target / record["path"])
return
_copy_new_tree(source, target, records, directories)
if (
manifest.inventory(target, path_prefix=path_prefix) != records
or manifest.directories(target, path_prefix=path_prefix) != directories
):
raise DeliveryError("installed image readback differs")
def _copy_new_tree(source: Path, target: Path, records: list[dict], directories: list[str]) -> None:
"""Create a new sealed tree; the caller must verify the complete destination."""
target.mkdir(mode=0o700)
for name in sorted(directories, key=lambda name: (name.count("/"), name)):
(target / name).mkdir(mode=0o700)
def copy(record: dict) -> None:
# The complete sealed tree is read back immediately below. Avoid
# hashing each destination both here and again in that inventory.
# frozen_copy verifies this image immediately; a release staging tree
# gets the stronger complete bundle check before atomic publication.
_copy_verified(
source / record["path"],
target / record["path"],
@@ -254,11 +264,6 @@ def frozen_copy(
sync_directory(target / name)
target.chmod(0o555)
sync_directory(target)
if (
manifest.inventory(target, path_prefix=path_prefix) != records
or manifest.directories(target, path_prefix=path_prefix) != directories
):
raise DeliveryError("installed image readback differs")
def files(bundle: Path, expected: str) -> dict:
@@ -304,8 +309,15 @@ def files(bundle: Path, expected: str) -> dict:
if stage.exists() or stage.is_symlink():
raise DeliveryError("incomplete release requires explicit operator inspection")
stage.mkdir(mode=0o700)
# `frozen_copy` accepts names through the same bounded allowlist.
frozen_copy(bundle / "payload", stage / "payload", verified_records=data["files"])
# Read every destination byte once in verify_staged, after copying the
# manifest and before publishing. Images still get their own complete
# independent readback in frozen_copy.
_copy_new_tree(
bundle / "payload",
stage / "payload",
data["files"],
manifest.directories(bundle / "payload"),
)
copy_verified(bundle / "manifest.json", stage / "manifest.json", expected, 0o444)
manifest.verify_staged(stage, expected)
stage.chmod(0o555)
@@ -332,11 +344,7 @@ def files(bundle: Path, expected: str) -> dict:
preflight.protected(legacy, directory=True)
if read_json(legacy / "manifest.json") != LEGACY_HEADER:
raise DeliveryError("legacy compatibility image header differs")
wanted = {
r["path"]: r
for r in manifest.inventory(image_base / "gnu", path_prefix="images/catalog/")
if r["path"] != "manifest.json"
}
wanted = {r["path"]: r for r in image_records["gnu"] if r["path"] != "manifest.json"}
actual = {
r["path"]: r
for r in manifest.inventory(legacy, path_prefix="images/catalog/")