feat(backup): capture private cold snapshots with reference verification
This commit is contained in:
@@ -0,0 +1,126 @@
|
||||
"""Read-only private backup inventory, separate from garbage collection and grading."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
from kf_oj.contracts.judging import ArtifactReference
|
||||
from kf_oj.core.settings import Settings
|
||||
from kf_oj.infra.artifacts.models import Artifact
|
||||
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||
|
||||
MAX_REFERENCES = 65536
|
||||
|
||||
|
||||
def verify_existing(root: Path, reference: ArtifactReference) -> None:
|
||||
"""No mkdir, lock creation or writable descriptor, even for a missing store."""
|
||||
root = root.absolute()
|
||||
if root.resolve(strict=True) != root:
|
||||
raise ValueError("backup storage path contains links")
|
||||
directory = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
try:
|
||||
info = os.fstat(directory)
|
||||
if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700:
|
||||
raise ValueError("backup storage root identity differs")
|
||||
descriptor = os.open(
|
||||
reference.digest,
|
||||
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||
dir_fd=directory,
|
||||
)
|
||||
with os.fdopen(descriptor, "rb") as source:
|
||||
before = os.fstat(source.fileno())
|
||||
if (
|
||||
not stat.S_ISREG(before.st_mode)
|
||||
or before.st_nlink != 1
|
||||
or before.st_uid != os.getuid()
|
||||
or stat.S_IMODE(before.st_mode) != 0o600
|
||||
or before.st_size != reference.bytes
|
||||
):
|
||||
raise ValueError("backup artifact identity or length differs")
|
||||
digest = hashlib.sha256()
|
||||
remaining = reference.bytes
|
||||
while remaining:
|
||||
block = source.read(min(65536, remaining))
|
||||
if not block:
|
||||
raise ValueError("backup artifact truncated")
|
||||
digest.update(block)
|
||||
remaining -= len(block)
|
||||
after = os.fstat(source.fileno())
|
||||
if (
|
||||
source.read(1)
|
||||
or digest.hexdigest() != reference.digest
|
||||
or any(
|
||||
getattr(before, key) != getattr(after, key)
|
||||
for key in ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns")
|
||||
)
|
||||
):
|
||||
raise ValueError("backup artifact changed or digest differs")
|
||||
finally:
|
||||
os.close(directory)
|
||||
|
||||
|
||||
async def inventory() -> dict:
|
||||
"""Every ready registry object covers all enforced ready-state foreign keys.
|
||||
|
||||
The operator separately proves a stopped-writing cut. This CLI neither
|
||||
claims that cut nor reads domain ORM models or changes a lifecycle state.
|
||||
"""
|
||||
settings = Settings()
|
||||
if settings.artifact_root is None:
|
||||
raise ValueError("artifact root required")
|
||||
engine = build_engine(settings)
|
||||
try:
|
||||
sessions = build_session_factory(engine)
|
||||
async with sessions.begin() as session:
|
||||
await session.execute(text("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY"))
|
||||
rows = await session.execute(
|
||||
select(Artifact.digest, Artifact.bytes)
|
||||
.where(Artifact.state == "ready")
|
||||
.order_by(Artifact.digest)
|
||||
.limit(MAX_REFERENCES + 1)
|
||||
)
|
||||
objects = list(rows)
|
||||
if len(objects) > MAX_REFERENCES:
|
||||
raise ValueError("backup registry count exceeds bound")
|
||||
references = []
|
||||
for digest, size in objects:
|
||||
reference = ArtifactReference(digest=digest, bytes=size)
|
||||
# A 64-KiB buffer checks actual bytes without source/case output.
|
||||
await asyncio.to_thread(verify_existing, settings.artifact_root, reference)
|
||||
references.append(reference.model_dump(mode="json"))
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"kind": "kf.private.artifact-backup-inventory",
|
||||
"read_only": True,
|
||||
"quiescence_verified": False,
|
||||
"boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(),
|
||||
"storage_identity": {
|
||||
"device": settings.artifact_root.stat().st_dev,
|
||||
"inode": settings.artifact_root.stat().st_ino,
|
||||
"uid": settings.artifact_root.stat().st_uid,
|
||||
},
|
||||
"objects": references,
|
||||
}
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Private read-only artifact backup inventory")
|
||||
parser.parse_args()
|
||||
try:
|
||||
result = asyncio.run(asyncio.wait_for(inventory(), timeout=300))
|
||||
print(json.dumps(result, separators=(",", ":"), sort_keys=True))
|
||||
except (ValueError, OSError, SQLAlchemyError, TimeoutError):
|
||||
parser.exit(1, "Backup artifact inventory refused; no metadata or grade was modified.\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Trusted adapter checks; real cold PG/role/permission tests remain VM-only."""
|
||||
|
||||
import hashlib
|
||||
|
||||
from kf_oj.contracts.judging import ArtifactReference
|
||||
from kf_oj.infra.artifacts.backup import verify_existing
|
||||
|
||||
|
||||
def test_backup_verifies_binary_and_empty_objects_without_creating_storage_entries(tmp_path):
|
||||
tmp_path.chmod(0o700)
|
||||
values = [b"", b"\0\xff\n" * 50000]
|
||||
expected = {}
|
||||
for value in values:
|
||||
digest = hashlib.sha256(value).hexdigest()
|
||||
path = tmp_path / digest
|
||||
path.write_bytes(value)
|
||||
path.chmod(0o600)
|
||||
expected[digest] = value
|
||||
before = {p.name: (p.read_bytes(), p.stat().st_mtime_ns) for p in tmp_path.iterdir()}
|
||||
for digest, value in expected.items():
|
||||
verify_existing(tmp_path, ArtifactReference(digest=digest, bytes=len(value)))
|
||||
after = {p.name: (p.read_bytes(), p.stat().st_mtime_ns) for p in tmp_path.iterdir()}
|
||||
assert after == before
|
||||
assert not (tmp_path / ".lock").exists()
|
||||
@@ -374,3 +374,45 @@ it does not substitute for actual browser CA/HTTPS/security acceptance in the VM
|
||||
The exact `/api/v1/health/live` route permits GET/HEAD for this bounded process
|
||||
liveness check. Other health, internal, execution and control routes remain
|
||||
outside the public proxy allowlist; liveness does not attest judging readiness.
|
||||
|
||||
Private cold backups use a separate format from public release bundles.
|
||||
`python3.13 -m kf_oj.infra.artifacts.backup` is an ordinary read-only artifact
|
||||
inventory command. Run it with the selected API database/storage identity after
|
||||
the owner-confirmed drain and while PostgreSQL is still available. It reads every
|
||||
ready artifact registry object through a repeatable-read, read-only transaction
|
||||
and hash-checks existing source/case bytes with a 64-KiB buffer. It creates no
|
||||
storage directory or lock and changes no metadata, grade, session or grant.
|
||||
Its output is private and does not itself prove that writers have stopped.
|
||||
|
||||
After the confirmed drain, stop the selected database and broker and verify all
|
||||
their processes have exited. `python3.13 -m scripts.release.backup capture
|
||||
--backup-id bk-<32-lowercase-hex> --references PRIVATE_INVENTORY
|
||||
--references-sha256 SHA` launches one fixed Root backup operator. It requires the
|
||||
same selected release/configuration, this boot's successful owner-derived cut,
|
||||
quiescent admission, empty task resources, no surviving service identities and
|
||||
clean PostgreSQL shutdown. The operator has only `CAP_DAC_READ_SEARCH`, no-new-
|
||||
privileges, 256-MiB memory, zero swap, 32 tasks, 50% CPU, a 256-MiB individual-file
|
||||
limit and a 30-minute deadline. Only the preprovisioned private backup volume is
|
||||
writable; this entry does not format/mount a volume or execute captured content.
|
||||
|
||||
Provision a separate Root-owned 0700 ext4 volume of 16 GiB at
|
||||
`/var/lib/kf-onlinejudge/backups`, with `nosuid,nodev,noexec` and no shared mount.
|
||||
The capture retains all source domains: the cold PostgreSQL cluster, source and
|
||||
testcase store, private import staging, all retained release/image versions,
|
||||
application/dependency/proxy configuration, factory replay history and operation
|
||||
receipts. Every entry has a bounded digest/size and original identity/mode;
|
||||
captured copies remain Root-only. Registered source/case references must all
|
||||
match captured files. Source modification, links, special files, unknown
|
||||
identities, insufficient space or an incomplete cut are refused. A failed target
|
||||
is retained for explicit inspection and never published as a successful backup.
|
||||
|
||||
The bound is 7 GiB and 50,000 entries per backup, at most two backup directories,
|
||||
with a seven-day declared retention period. Capacity refusal does not silently
|
||||
delete an old/unknown backup. `backup verify --backup-id ID
|
||||
--manifest-sha256 SHA` checks an explicitly supplied private operator receipt and
|
||||
every private member; it performs no restoration or secret/epoch reuse. Actual
|
||||
retention pruning, controlled recovery into a clean node and measured recovery
|
||||
time require their separate implementation and independent VM acceptance.
|
||||
Backup data and manifests contain private credentials/source/testcase data and
|
||||
must stay outside Git and CI artifacts. Parsing, ordinary adapter tests or a
|
||||
manifest digest are not cold-backup/recovery acceptance evidence.
|
||||
|
||||
@@ -0,0 +1,504 @@
|
||||
"""Explicit cold private backup; stopped owner proofs precede any data capture."""
|
||||
|
||||
import argparse
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import resource
|
||||
import stat
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import (
|
||||
availability,
|
||||
backup_manifest,
|
||||
configuration,
|
||||
install,
|
||||
lifecycle,
|
||||
preflight,
|
||||
service_install,
|
||||
)
|
||||
from .policy import DeliveryError, canonical, digest, read_json, sha256
|
||||
from .services import python_environment
|
||||
|
||||
ROOT = Path("/var/lib/kf-onlinejudge/backups")
|
||||
CAPACITY = 16 * 1024**3
|
||||
SOURCES = {
|
||||
"postgres": Path("/var/lib/kf-onlinejudge/postgres/data"),
|
||||
"artifacts": Path("/var/lib/kf-onlinejudge/artifacts"),
|
||||
"imports": Path("/var/lib/kf-onlinejudge/imports"),
|
||||
"app-config": Path("/etc/kf-onlinejudge"),
|
||||
"judge-config": Path("/etc/kf-judge"),
|
||||
"factory": Path("/var/lib/kf-judge/factory"),
|
||||
"operations": lifecycle.STATE,
|
||||
"releases": install.RELEASES,
|
||||
"images": install.IMAGES,
|
||||
"dependencies": Path("/etc/kf-dependencies"),
|
||||
"proxy": Path("/etc/kf-proxy"),
|
||||
}
|
||||
|
||||
|
||||
def identifier(value: str) -> str:
|
||||
if not isinstance(value, str) or re.fullmatch(r"bk-[0-9a-f]{32}", value) is None:
|
||||
raise DeliveryError("explicit private backup ID required")
|
||||
return value
|
||||
|
||||
|
||||
def manager(*args: str) -> dict:
|
||||
raw = service_install.systemctl(*args)
|
||||
return dict(line.split("=", 1) for line in raw.splitlines() if "=" in line)
|
||||
|
||||
|
||||
def stopped(selected: dict, cut: dict) -> None:
|
||||
lifecycle.validate_record(cut)
|
||||
if (
|
||||
cut["source_sha"][:12] != selected["release_id"][3:]
|
||||
or cut["configuration_id"] != selected["configuration_id"]
|
||||
or cut["boot_id"] != Path("/proc/sys/kernel/random/boot_id").read_text().strip()
|
||||
or cut["action"] not in {"drain", "stop"}
|
||||
or cut["phase"] != "complete"
|
||||
or cut["success"] is not True
|
||||
or cut["root_cleanup_confirmed"] is not True
|
||||
or "drain_projection" not in cut
|
||||
or not lifecycle.projection_empty(cut["drain_projection"])
|
||||
):
|
||||
raise DeliveryError("cold backup requires this boot owner-confirmed drained cut")
|
||||
record = availability.read()
|
||||
if (
|
||||
record is None
|
||||
or record.release_id != selected["release_id"]
|
||||
or record.configuration_id != selected["configuration_id"]
|
||||
or record.state != "quiescent"
|
||||
):
|
||||
raise DeliveryError("cold backup admission is not proven quiescent")
|
||||
for name in [*lifecycle.APPS, *lifecycle.DEPENDENCIES]:
|
||||
props = manager(
|
||||
"show",
|
||||
"kf-" + name + ".service",
|
||||
"--property=MainPID,ActiveState,ControlGroup",
|
||||
)
|
||||
if props.get("MainPID") != "0" or props.get("ActiveState") != "inactive":
|
||||
raise DeliveryError("cold backup has an active required service")
|
||||
if props.get("ControlGroup"):
|
||||
group = Path("/sys/fs/cgroup") / props["ControlGroup"].lstrip("/")
|
||||
if group.exists() and (group / "cgroup.procs").read_text().strip():
|
||||
raise DeliveryError("cold backup service processes remain")
|
||||
for name in ("tasks", "identities", "captures"):
|
||||
path = Path("/var/lib/kf-judge") / name
|
||||
if path.exists() and next(path.iterdir(), None) is not None:
|
||||
raise DeliveryError("cold backup task resources remain")
|
||||
if Path("/var/lib/kf-judge/factory/control.sock").exists():
|
||||
raise DeliveryError("cold backup factory socket remains")
|
||||
processes = [p for p in Path("/proc").iterdir() if p.name.isdecimal()]
|
||||
if len(processes) > 1024:
|
||||
raise DeliveryError("cold backup process observation exceeds bound")
|
||||
for p in processes:
|
||||
try:
|
||||
status = (p / "status").read_text()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
rows = [line.split()[1:] for line in status.splitlines() if line.startswith("Uid:")]
|
||||
if len(rows) != 1 or len(rows[0]) != 4:
|
||||
raise DeliveryError("cold backup process identity unavailable")
|
||||
if any(int(uid) in {1200, 1201, 1202, 1203, 1204, 1205, 100000} for uid in rows[0]):
|
||||
raise DeliveryError("cold backup unmanaged data writer or task remains")
|
||||
if (SOURCES["postgres"] / "postmaster.pid").exists():
|
||||
raise DeliveryError("PostgreSQL did not confirm clean cold shutdown")
|
||||
|
||||
|
||||
def volume() -> None:
|
||||
preflight.protected(ROOT, directory=True)
|
||||
if stat.S_IMODE(ROOT.stat().st_mode) != 0o700:
|
||||
raise DeliveryError("private preprovisioned backup volume required")
|
||||
rows = [line.split() for line in Path("/proc/self/mountinfo").read_text().splitlines()]
|
||||
matches = [r for r in rows if r[4] == str(ROOT)]
|
||||
if len(matches) != 1:
|
||||
raise DeliveryError("separate hard-sized backup filesystem required")
|
||||
row = matches[0]
|
||||
sep = row.index("-")
|
||||
opts = set(row[5].split(",")) | set(row[sep + 3].split(","))
|
||||
fs = os.statvfs(ROOT)
|
||||
if (
|
||||
row[3] != "/"
|
||||
or row[sep + 1] != "ext4"
|
||||
or any(r.startswith(("shared:", "master:")) for r in row[6:sep])
|
||||
or not {"rw", "nosuid", "nodev", "noexec"} <= opts
|
||||
or "ro" in opts
|
||||
or not CAPACITY * 9 // 10 <= fs.f_frsize * fs.f_blocks <= CAPACITY
|
||||
):
|
||||
raise DeliveryError("backup volume limits or mount isolation differ")
|
||||
members = list(ROOT.iterdir())
|
||||
snapshots = []
|
||||
for p in members:
|
||||
preflight.protected(p, directory=p.is_dir())
|
||||
if p.name == "lost+found" and p.is_dir() and not list(p.iterdir()):
|
||||
continue
|
||||
identifier(p.name)
|
||||
if not p.is_dir() or stat.S_IMODE(p.stat().st_mode) != 0o700:
|
||||
raise DeliveryError("unknown private backup volume member")
|
||||
snapshots.append(p)
|
||||
if len(snapshots) >= 2:
|
||||
raise DeliveryError(
|
||||
"two-backup retention capacity reached; explicit verified pruning required"
|
||||
)
|
||||
|
||||
|
||||
def guarded() -> None:
|
||||
if (os.getuid(), os.geteuid(), os.getgid()) != (0, 0, 0) or not set(os.getgroups()) <= {0}:
|
||||
raise DeliveryError("root backup operator required")
|
||||
status = dict(
|
||||
line.split(":", 1)
|
||||
for line in Path("/proc/self/status").read_text().splitlines()
|
||||
if ":" in line
|
||||
)
|
||||
if status.get("NoNewPrivs", "").strip() != "1" or any(
|
||||
int(status.get(k, "-1").strip(), 16) != 4
|
||||
for k in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb")
|
||||
):
|
||||
raise DeliveryError("fixed read-search-only backup capability required")
|
||||
group = preflight.current_cgroup()
|
||||
if group != Path("/sys/fs/cgroup/kfonlinejudge.slice/kf-node-backup.service"):
|
||||
raise DeliveryError("fixed backup operator unit required")
|
||||
if {
|
||||
k: (group / k).read_text().strip()
|
||||
for k in ("memory.max", "memory.swap.max", "pids.max", "cpu.max")
|
||||
} != {
|
||||
"memory.max": "268435456",
|
||||
"memory.swap.max": "0",
|
||||
"pids.max": "32",
|
||||
"cpu.max": "50000 100000",
|
||||
}:
|
||||
raise DeliveryError("actual backup operator limits differ")
|
||||
node = Path("/sys/fs/cgroup/kfonlinejudge.slice")
|
||||
if {
|
||||
k: (node / k).read_text().strip()
|
||||
for k in ("memory.max", "memory.swap.max", "pids.max", "cpu.max")
|
||||
} != {
|
||||
"memory.max": "4026531840",
|
||||
"memory.swap.max": "0",
|
||||
"pids.max": "384",
|
||||
"cpu.max": "200000 100000",
|
||||
}:
|
||||
raise DeliveryError("actual backup node limits differ")
|
||||
for key, bound in (
|
||||
(resource.RLIMIT_CORE, 0),
|
||||
(resource.RLIMIT_FSIZE, backup_manifest.MAX_FILE_BYTES),
|
||||
(resource.RLIMIT_NOFILE, 256),
|
||||
(resource.RLIMIT_STACK, 16 * 1024**2),
|
||||
):
|
||||
if resource.getrlimit(key) != (bound, bound):
|
||||
raise DeliveryError("backup process limit differs")
|
||||
|
||||
|
||||
def capture(backup_id: str, references_file: Path, references_sha: str) -> dict:
|
||||
guarded()
|
||||
selected = install.selection()
|
||||
payload = install.RELEASES / selected["release_id"] / "payload"
|
||||
for name in ("backup.py", "backup_manifest.py"):
|
||||
if digest(payload / "delivery/scripts/release" / name) != digest(
|
||||
Path(__file__).parent / name
|
||||
):
|
||||
raise DeliveryError("backup operator differs from selected release")
|
||||
configuration.private(references_file)
|
||||
if digest(references_file) != sha256(references_sha):
|
||||
raise DeliveryError("separate private artifact receipt differs")
|
||||
refs = backup_manifest.validate_references(backup_manifest.read_metadata(references_file))
|
||||
cut = read_json(lifecycle.LAST, 32768)
|
||||
stopped(selected, cut)
|
||||
artifact_root = SOURCES["artifacts"].stat()
|
||||
if refs["boot_id"] != cut["boot_id"] or refs["storage_identity"] != {
|
||||
"device": artifact_root.st_dev,
|
||||
"inode": artifact_root.st_ino,
|
||||
"uid": artifact_root.st_uid,
|
||||
}:
|
||||
raise DeliveryError("artifact inventory was not observed on this source storage/boot")
|
||||
volume()
|
||||
lock = lifecycle.STATE / "maintenance.lock"
|
||||
configuration.private(lock)
|
||||
descriptor = os.open(lock, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
deadline = time.monotonic() + 1700
|
||||
target = ROOT / identifier(backup_id)
|
||||
try:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
stopped(selected, cut)
|
||||
if target.exists() or target.is_symlink():
|
||||
raise DeliveryError("backup target already exists; incomplete backups retained")
|
||||
target.mkdir(mode=0o700)
|
||||
tree = target / "payload"
|
||||
tree.mkdir(mode=0o700)
|
||||
dirs, files, total = [], [], 0
|
||||
for domain, source_root in SOURCES.items():
|
||||
if source_root.is_symlink() or not source_root.is_dir():
|
||||
raise DeliveryError("a required backup source domain is absent or linked")
|
||||
for directory, names, members in os.walk(source_root, followlinks=False):
|
||||
if time.monotonic() >= deadline:
|
||||
raise DeliveryError("private backup capture deadline")
|
||||
source_dir = Path(directory)
|
||||
relative = backup_manifest.name(
|
||||
domain
|
||||
+ (
|
||||
"/" + source_dir.relative_to(source_root).as_posix()
|
||||
if source_dir != source_root
|
||||
else ""
|
||||
)
|
||||
)
|
||||
info = source_dir.lstat()
|
||||
entry = {
|
||||
"path": relative,
|
||||
"uid": info.st_uid,
|
||||
"gid": info.st_gid,
|
||||
"mode": stat.S_IMODE(info.st_mode),
|
||||
}
|
||||
backup_manifest._member(entry, set(), directory=True)
|
||||
if not stat.S_ISDIR(info.st_mode):
|
||||
raise DeliveryError("backup source directory changed type")
|
||||
destination = tree / relative
|
||||
destination.mkdir(mode=0o700)
|
||||
dirs.append(entry)
|
||||
if len(dirs) + len(files) > backup_manifest.MAX_ENTRIES:
|
||||
raise DeliveryError("backup directory inventory exceeds bound")
|
||||
for member in names:
|
||||
if (source_dir / member).is_symlink():
|
||||
raise DeliveryError("linked backup directory refused")
|
||||
for member in sorted(members):
|
||||
p = source_dir / member
|
||||
before = p.lstat()
|
||||
if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1:
|
||||
raise DeliveryError("special or linked backup source refused")
|
||||
entry = {
|
||||
"path": backup_manifest.name(relative + "/" + member),
|
||||
"uid": before.st_uid,
|
||||
"gid": before.st_gid,
|
||||
"mode": stat.S_IMODE(before.st_mode),
|
||||
"bytes": before.st_size,
|
||||
}
|
||||
backup_manifest._member(entry, set(), directory=False)
|
||||
if before.st_size > backup_manifest.MAX_FILE_BYTES:
|
||||
raise DeliveryError("backup individual file exceeds bound")
|
||||
total += before.st_size
|
||||
if (
|
||||
total > backup_manifest.MAX_BYTES
|
||||
or len(dirs) + len(files) >= backup_manifest.MAX_ENTRIES
|
||||
or os.statvfs(ROOT).f_frsize * os.statvfs(ROOT).f_bavail
|
||||
< before.st_size + 64 * 1024**2
|
||||
):
|
||||
raise DeliveryError("private backup capacity exhausted")
|
||||
fd = os.open(p, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC)
|
||||
h = hashlib.sha256()
|
||||
output = os.open(
|
||||
destination / member,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
0o600,
|
||||
)
|
||||
with (
|
||||
os.fdopen(fd, "rb") as incoming,
|
||||
os.fdopen(output, "wb") as outgoing,
|
||||
):
|
||||
observed = os.fstat(incoming.fileno())
|
||||
if any(
|
||||
getattr(before, k) != getattr(observed, k)
|
||||
for k in (
|
||||
"st_dev",
|
||||
"st_ino",
|
||||
"st_size",
|
||||
"st_mtime_ns",
|
||||
"st_ctime_ns",
|
||||
)
|
||||
):
|
||||
raise DeliveryError("backup source changed before capture")
|
||||
count = 0
|
||||
while block := incoming.read(65536):
|
||||
count += len(block)
|
||||
if count > before.st_size or time.monotonic() >= deadline:
|
||||
raise DeliveryError("backup source or deadline changed")
|
||||
h.update(block)
|
||||
outgoing.write(block)
|
||||
after = os.fstat(incoming.fileno())
|
||||
if count != before.st_size or any(
|
||||
getattr(before, k) != getattr(after, k)
|
||||
for k in (
|
||||
"st_dev",
|
||||
"st_ino",
|
||||
"st_size",
|
||||
"st_mtime_ns",
|
||||
"st_ctime_ns",
|
||||
)
|
||||
):
|
||||
raise DeliveryError("backup source changed during capture")
|
||||
outgoing.flush()
|
||||
os.fsync(outgoing.fileno())
|
||||
os.fchmod(outgoing.fileno(), 0o600)
|
||||
entry["sha256"] = h.hexdigest()
|
||||
files.append(entry)
|
||||
install.sync_directory(destination)
|
||||
stopped(selected, cut)
|
||||
if install.selection() != selected or read_json(lifecycle.LAST, 32768) != cut:
|
||||
raise DeliveryError("selected cut changed during backup")
|
||||
manifest = read_json(payload.parent / "manifest.json")
|
||||
now = int(time.time())
|
||||
value = backup_manifest.validate(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"format": backup_manifest.FORMAT,
|
||||
"backup_id": backup_id,
|
||||
"source_sha": manifest["source_sha"],
|
||||
"manifest_sha256": selected["manifest_sha256"],
|
||||
"configuration_id": selected["configuration_id"],
|
||||
"created_unix_seconds": now,
|
||||
"expires_unix_seconds": now + 7 * 86400,
|
||||
"consistent_cut": cut,
|
||||
"selected": selected,
|
||||
"artifact_references": refs,
|
||||
"images": manifest["images"],
|
||||
"directories": sorted(dirs, key=lambda x: x["path"]),
|
||||
"files": sorted(files, key=lambda x: x["path"]),
|
||||
"payload_bytes": total,
|
||||
"restored": False,
|
||||
"execution_verified": False,
|
||||
}
|
||||
)
|
||||
raw = canonical(value)
|
||||
if len(raw) > backup_manifest.MAX_METADATA_BYTES:
|
||||
raise DeliveryError("private backup manifest exceeds bound")
|
||||
install.exclusive_file(target / "manifest.json", raw, 0o600)
|
||||
install.sync_directory(tree)
|
||||
install.sync_directory(target)
|
||||
install.sync_directory(ROOT)
|
||||
fingerprint = digest(target / "manifest.json")
|
||||
backup_manifest.verify(target, fingerprint)
|
||||
return {
|
||||
"backup_id": backup_id,
|
||||
"private_manifest_sha256": fingerprint,
|
||||
"payload_bytes": total,
|
||||
"source_sha": manifest["source_sha"],
|
||||
"captured": True,
|
||||
"restored": False,
|
||||
"execution_verified": False,
|
||||
}
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Private bounded cold backup operator")
|
||||
sub = parser.add_subparsers(dest="action", required=True)
|
||||
create = sub.add_parser("capture")
|
||||
create.add_argument("--backup-id", required=True)
|
||||
create.add_argument("--references", type=Path, required=True)
|
||||
create.add_argument("--references-sha256", required=True)
|
||||
create.add_argument("--inside", action="store_true")
|
||||
check = sub.add_parser("verify")
|
||||
check.add_argument("--backup-id", required=True)
|
||||
check.add_argument("--manifest-sha256", required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if os.getuid() != 0 or os.geteuid() != 0:
|
||||
raise DeliveryError("explicit root backup operator required")
|
||||
if args.action == "verify":
|
||||
value = backup_manifest.verify(ROOT / identifier(args.backup_id), args.manifest_sha256)
|
||||
result = {
|
||||
"verified": True,
|
||||
"backup_id": value["backup_id"],
|
||||
"restored": False,
|
||||
"execution_verified": False,
|
||||
}
|
||||
elif args.inside:
|
||||
result = capture(args.backup_id, args.references, args.references_sha256)
|
||||
else:
|
||||
# Caller provides only an ID/private digest; no program, identity,
|
||||
# source-domain or resource override is accepted by this launcher.
|
||||
identifier(args.backup_id)
|
||||
configuration.private(args.references)
|
||||
if digest(args.references) != sha256(args.references_sha256):
|
||||
raise DeliveryError("private artifact inventory receipt differs")
|
||||
selected = install.selection()
|
||||
payload = install.RELEASES / selected["release_id"] / "payload"
|
||||
current = manager(
|
||||
"show",
|
||||
"kf-node-backup.service",
|
||||
"--property=LoadState,ActiveState,MainPID",
|
||||
)
|
||||
if current != {
|
||||
"LoadState": "not-found",
|
||||
"ActiveState": "inactive",
|
||||
"MainPID": "0",
|
||||
}:
|
||||
raise DeliveryError("unknown or active backup operator")
|
||||
command = [
|
||||
"/usr/bin/systemd-run",
|
||||
"--quiet",
|
||||
"--wait",
|
||||
"--pipe",
|
||||
"--collect",
|
||||
"--unit=kf-node-backup.service",
|
||||
"--slice=kfonlinejudge.slice",
|
||||
]
|
||||
for key, value in {
|
||||
"User": "root",
|
||||
"Group": "root",
|
||||
"MemoryMax": "256M",
|
||||
"MemorySwapMax": "0",
|
||||
"TasksMax": "32",
|
||||
"CPUQuota": "50%",
|
||||
"RuntimeMaxSec": "1800",
|
||||
"TimeoutStopSec": "15",
|
||||
"KillMode": "control-group",
|
||||
"Restart": "no",
|
||||
"LimitCORE": "0",
|
||||
"LimitFSIZE": "256M",
|
||||
"LimitNOFILE": "256",
|
||||
"LimitSTACK": "16M",
|
||||
"UMask": "0077",
|
||||
"NoNewPrivileges": "yes",
|
||||
"CapabilityBoundingSet": "CAP_DAC_READ_SEARCH",
|
||||
"AmbientCapabilities": "CAP_DAC_READ_SEARCH",
|
||||
"ProtectSystem": "strict",
|
||||
"ProtectHome": "yes",
|
||||
"PrivateTmp": "yes",
|
||||
"PrivateDevices": "yes",
|
||||
"ProtectControlGroups": "yes",
|
||||
"RestrictAddressFamilies": "AF_UNIX",
|
||||
"ReadWritePaths": str(ROOT),
|
||||
}.items():
|
||||
command += ["--property=" + key + "=" + value]
|
||||
command += [
|
||||
"/usr/bin/env",
|
||||
"-i",
|
||||
"PATH=/usr/bin:/bin",
|
||||
"LC_ALL=C.UTF-8",
|
||||
*python_environment(payload).split(),
|
||||
str(payload / "python/bin/python3.13"),
|
||||
"-s",
|
||||
"-P",
|
||||
"-m",
|
||||
"scripts.release.backup",
|
||||
"capture",
|
||||
"--inside",
|
||||
"--backup-id",
|
||||
args.backup_id,
|
||||
"--references",
|
||||
str(args.references),
|
||||
"--references-sha256",
|
||||
args.references_sha256,
|
||||
]
|
||||
result = subprocess.run(command, capture_output=True, timeout=1820, check=False)
|
||||
if result.returncode or len(result.stdout) > 4096:
|
||||
raise DeliveryError(
|
||||
"cold backup capture failed; source remains stopped and partial target retained"
|
||||
)
|
||||
result = json.loads(result.stdout)
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
except (OSError, ValueError, subprocess.SubprocessError) as error:
|
||||
parser.exit(
|
||||
1,
|
||||
"Private backup refused: "
|
||||
+ (str(error) if isinstance(error, DeliveryError) else type(error).__name__)
|
||||
+ "\n",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,341 @@
|
||||
"""Private cold-backup directory format; never a public release or CI artifact."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
from pathlib import Path
|
||||
|
||||
from .policy import (
|
||||
DeliveryError,
|
||||
digest,
|
||||
fields,
|
||||
integer,
|
||||
invalid_constant,
|
||||
sha256,
|
||||
unique_object,
|
||||
)
|
||||
|
||||
FORMAT = "kf.private.cold-backup.v1"
|
||||
MAX_BYTES = 7 * 1024**3
|
||||
MAX_FILE_BYTES = 256 * 1024**2
|
||||
MAX_ENTRIES = 50000
|
||||
MAX_METADATA_BYTES = 16 * 1024**2
|
||||
DOMAINS = {
|
||||
"postgres",
|
||||
"artifacts",
|
||||
"imports",
|
||||
"app-config",
|
||||
"judge-config",
|
||||
"factory",
|
||||
"operations",
|
||||
"releases",
|
||||
"images",
|
||||
"dependencies",
|
||||
"proxy",
|
||||
}
|
||||
|
||||
|
||||
def read_metadata(path: Path) -> dict:
|
||||
"""Private inventories can exceed the public bundle's structure count.
|
||||
|
||||
Byte, depth and entry bounds remain explicit before any restore/capture use.
|
||||
"""
|
||||
with path.open("rb") as stream:
|
||||
raw = stream.read(MAX_METADATA_BYTES + 1)
|
||||
if len(raw) > MAX_METADATA_BYTES:
|
||||
raise DeliveryError("private backup metadata byte limit")
|
||||
try:
|
||||
value = json.loads(raw, object_pairs_hook=unique_object, parse_constant=invalid_constant)
|
||||
pending = [(value, 0)]
|
||||
count = 0
|
||||
while pending:
|
||||
item, depth = pending.pop()
|
||||
count += 1
|
||||
if depth > 32 or count > 1_000_000:
|
||||
raise DeliveryError("private backup metadata structure limit")
|
||||
if isinstance(item, dict):
|
||||
pending.extend((member, depth + 1) for member in item.values())
|
||||
elif isinstance(item, list):
|
||||
pending.extend((member, depth + 1) for member in item)
|
||||
return value
|
||||
except (UnicodeError, json.JSONDecodeError, RecursionError) as error:
|
||||
raise DeliveryError("invalid private backup metadata") from error
|
||||
|
||||
|
||||
def name(value: str) -> str:
|
||||
if not isinstance(value, str) or not 1 <= len(value) <= 512:
|
||||
raise DeliveryError("invalid private backup member")
|
||||
parts = value.split("/")
|
||||
if parts[0] not in DOMAINS or any(
|
||||
p in {"", ".", "..", ".git", ".ssh", ".localplan", "docs"}
|
||||
or re.fullmatch(r"[A-Za-z0-9_.+@-]{1,255}", p) is None
|
||||
for p in parts
|
||||
):
|
||||
raise DeliveryError("private backup member outside fixed domains")
|
||||
return value
|
||||
|
||||
|
||||
def validate_references(value: dict) -> dict:
|
||||
fields(
|
||||
value,
|
||||
{
|
||||
"schema_version",
|
||||
"kind",
|
||||
"read_only",
|
||||
"quiescence_verified",
|
||||
"boot_id",
|
||||
"storage_identity",
|
||||
"objects",
|
||||
},
|
||||
"private artifact references",
|
||||
)
|
||||
if (
|
||||
type(value["schema_version"]) is not int
|
||||
or value["schema_version"] != 1
|
||||
or value["kind"] != "kf.private.artifact-backup-inventory"
|
||||
or value["read_only"] is not True
|
||||
or value["quiescence_verified"] is not False
|
||||
or not isinstance(value["objects"], list)
|
||||
or len(value["objects"]) > 65536
|
||||
):
|
||||
raise DeliveryError("artifact backup inventory authority differs")
|
||||
from uuid import UUID
|
||||
|
||||
if (
|
||||
not isinstance(value["boot_id"], str)
|
||||
or str(UUID(value["boot_id"])) != value["boot_id"]
|
||||
or UUID(value["boot_id"]).int == 0
|
||||
):
|
||||
raise DeliveryError("artifact backup boot identity differs")
|
||||
identity = fields(
|
||||
value["storage_identity"],
|
||||
{"device", "inode", "uid"},
|
||||
"artifact storage identity",
|
||||
)
|
||||
integer(identity["device"], 1, 2**64 - 1, "artifact storage device")
|
||||
integer(identity["inode"], 1, 2**64 - 1, "artifact storage inode")
|
||||
if type(identity["uid"]) is not int or identity["uid"] != 1200:
|
||||
raise DeliveryError("artifact storage identity is not the platform owner")
|
||||
seen = set()
|
||||
for item in value["objects"]:
|
||||
fields(item, {"digest", "bytes"}, "private artifact reference")
|
||||
sha256(item["digest"])
|
||||
integer(item["bytes"], 0, 64 * 1024**2, "artifact reference bytes")
|
||||
if item["digest"] in seen:
|
||||
raise DeliveryError("duplicate backup artifact reference")
|
||||
seen.add(item["digest"])
|
||||
return value
|
||||
|
||||
|
||||
def validate(value: dict) -> dict:
|
||||
fields(
|
||||
value,
|
||||
{
|
||||
"schema_version",
|
||||
"format",
|
||||
"backup_id",
|
||||
"source_sha",
|
||||
"manifest_sha256",
|
||||
"configuration_id",
|
||||
"created_unix_seconds",
|
||||
"expires_unix_seconds",
|
||||
"consistent_cut",
|
||||
"selected",
|
||||
"artifact_references",
|
||||
"images",
|
||||
"directories",
|
||||
"files",
|
||||
"payload_bytes",
|
||||
"restored",
|
||||
"execution_verified",
|
||||
},
|
||||
"private backup",
|
||||
)
|
||||
if (
|
||||
type(value["schema_version"]) is not int
|
||||
or value["schema_version"] != 1
|
||||
or value["format"] != FORMAT
|
||||
or not isinstance(value["backup_id"], str)
|
||||
or re.fullmatch(r"bk-[0-9a-f]{32}", value["backup_id"]) is None
|
||||
or not isinstance(value["source_sha"], str)
|
||||
or re.fullmatch(r"[0-9a-f]{40}", value["source_sha"]) is None
|
||||
or not isinstance(value["configuration_id"], str)
|
||||
or re.fullmatch(r"cfg-[0-9a-f]{24}", value["configuration_id"]) is None
|
||||
or value["restored"] is not False
|
||||
or value["execution_verified"] is not False
|
||||
):
|
||||
raise DeliveryError("private backup identity or authority differs")
|
||||
sha256(value["manifest_sha256"])
|
||||
start = integer(value["created_unix_seconds"], 1, 2**63 - 1, "backup time")
|
||||
end = integer(value["expires_unix_seconds"], start + 1, start + 7 * 86400, "backup retention")
|
||||
if end - start != 7 * 86400:
|
||||
raise DeliveryError("fixed seven-day private backup retention required")
|
||||
from .lifecycle import projection_empty, validate_record
|
||||
|
||||
cut = validate_record(value["consistent_cut"])
|
||||
if (
|
||||
cut["source_sha"] != value["source_sha"]
|
||||
or cut["configuration_id"] != value["configuration_id"]
|
||||
or cut["action"] not in {"drain", "stop"}
|
||||
or cut["phase"] != "complete"
|
||||
or cut["success"] is not True
|
||||
or cut["root_cleanup_confirmed"] is not True
|
||||
or "drain_projection" not in cut
|
||||
or not projection_empty(cut["drain_projection"])
|
||||
):
|
||||
raise DeliveryError("cold backup lacks a confirmed owner-derived cut")
|
||||
selected = fields(
|
||||
value["selected"],
|
||||
{
|
||||
"schema_version",
|
||||
"release_id",
|
||||
"manifest_sha256",
|
||||
"configuration_id",
|
||||
"receipt_sha256",
|
||||
},
|
||||
"backup selected binding",
|
||||
)
|
||||
if (
|
||||
type(selected["schema_version"]) is not int
|
||||
or selected["schema_version"] != 1
|
||||
or selected["release_id"] != "sd-" + value["source_sha"][:12]
|
||||
or selected["manifest_sha256"] != value["manifest_sha256"]
|
||||
or selected["configuration_id"] != value["configuration_id"]
|
||||
):
|
||||
raise DeliveryError("backup selected binding differs")
|
||||
sha256(selected["receipt_sha256"])
|
||||
validate_references(value["artifact_references"])
|
||||
if value["artifact_references"]["boot_id"] != cut["boot_id"]:
|
||||
raise DeliveryError("artifact snapshot and cold cut are from different boots")
|
||||
fields(value["images"], {"gnu", "python313", "java21"}, "backup image identities")
|
||||
for identity in value["images"].values():
|
||||
fields(
|
||||
identity,
|
||||
{
|
||||
"toolchain_id",
|
||||
"image_kind",
|
||||
"image_revision",
|
||||
"runtime_version",
|
||||
"image_digest",
|
||||
},
|
||||
"backup runtime identity",
|
||||
)
|
||||
sha256(identity["image_digest"])
|
||||
integer(identity["image_revision"], 1, 2**32 - 1, "backup image revision")
|
||||
for key in ("toolchain_id", "image_kind", "runtime_version"):
|
||||
if (
|
||||
not isinstance(identity[key], str)
|
||||
or re.fullmatch(r"[A-Za-z0-9_.+-]{1,128}", identity[key]) is None
|
||||
):
|
||||
raise DeliveryError("backup runtime label differs")
|
||||
dirs, files = value["directories"], value["files"]
|
||||
if (
|
||||
not isinstance(dirs, list)
|
||||
or not isinstance(files, list)
|
||||
or not 1 <= len(dirs) + len(files) <= MAX_ENTRIES
|
||||
):
|
||||
raise DeliveryError("backup inventory count exceeds bound")
|
||||
seen = set()
|
||||
for item in dirs:
|
||||
fields(item, {"path", "uid", "gid", "mode"}, "backup directory")
|
||||
_member(item, seen, directory=True)
|
||||
total = 0
|
||||
for item in files:
|
||||
fields(item, {"path", "uid", "gid", "mode", "bytes", "sha256"}, "backup file")
|
||||
_member(item, seen, directory=False)
|
||||
total += integer(item["bytes"], 0, MAX_FILE_BYTES, "backup file bytes")
|
||||
sha256(item["sha256"])
|
||||
if (
|
||||
type(value["payload_bytes"]) is not int
|
||||
or value["payload_bytes"] != total
|
||||
or total > MAX_BYTES
|
||||
):
|
||||
raise DeliveryError("private backup aggregate bytes differ")
|
||||
directories = {item["path"] for item in dirs}
|
||||
if not DOMAINS <= directories:
|
||||
raise DeliveryError("private backup omitted a required source domain")
|
||||
for path in seen:
|
||||
for parent in Path(path).parents:
|
||||
if parent.as_posix() != "." and parent.as_posix() not in directories:
|
||||
raise DeliveryError("private backup parent missing")
|
||||
lookup = {item["path"]: item for item in files}
|
||||
for reference in value["artifact_references"]["objects"]:
|
||||
item = lookup.get("artifacts/" + reference["digest"])
|
||||
if (
|
||||
item is None
|
||||
or item["bytes"] != reference["bytes"]
|
||||
or item["sha256"] != reference["digest"]
|
||||
):
|
||||
raise DeliveryError("registered source or testcase absent from backup")
|
||||
if "postgres/PG_VERSION" not in lookup or "postgres/global/pg_control" not in lookup:
|
||||
raise DeliveryError("private backup lacks a cold PostgreSQL cluster")
|
||||
return value
|
||||
|
||||
|
||||
def _member(item: dict, seen: set[str], *, directory: bool) -> None:
|
||||
path = name(item["path"])
|
||||
if path in seen:
|
||||
raise DeliveryError("duplicate private backup member")
|
||||
seen.add(path)
|
||||
integer(item["uid"], 0, 1205, "backup member uid")
|
||||
integer(item["gid"], 0, 1205, "backup member gid")
|
||||
if item["uid"] not in {0, 1200, 1201, 1202, 1203, 1204, 1205} or (
|
||||
item["gid"] not in ({0, 1203, 1204, 1205} if item["uid"] == 0 else {item["uid"]})
|
||||
):
|
||||
raise DeliveryError("unsupported backup OS identity")
|
||||
mode = item["mode"]
|
||||
if type(mode) is not int or mode not in (
|
||||
{0o555, 0o700, 0o711, 0o755}
|
||||
if directory
|
||||
else {0o400, 0o444, 0o600, 0o640, 0o644, 0o555, 0o755}
|
||||
):
|
||||
raise DeliveryError("backup member privilege bits differ")
|
||||
|
||||
|
||||
def verify(root: Path, expected: str) -> dict:
|
||||
"""Expected digest is a separate private operator receipt, never self-trust."""
|
||||
from . import configuration, preflight
|
||||
|
||||
preflight.protected(root, directory=True)
|
||||
if stat.S_IMODE(root.stat().st_mode) != 0o700:
|
||||
raise DeliveryError("private backup directory required")
|
||||
path = root / "manifest.json"
|
||||
configuration.private(path)
|
||||
if digest(path) != sha256(expected) or {p.name for p in root.iterdir()} != {
|
||||
"manifest.json",
|
||||
"payload",
|
||||
}:
|
||||
raise DeliveryError("private backup receipt or layout differs")
|
||||
value = validate(read_metadata(path))
|
||||
found_files, found_dirs = set(), set()
|
||||
payload = root / "payload"
|
||||
preflight.protected(payload, directory=True)
|
||||
for directory, names, files in os.walk(payload, followlinks=False):
|
||||
for member in names + files:
|
||||
p = Path(directory) / member
|
||||
info = p.lstat()
|
||||
relative = name(p.relative_to(payload).as_posix())
|
||||
if info.st_uid != 0 or info.st_gid != 0 or p.is_symlink():
|
||||
raise DeliveryError("private backup member ownership differs")
|
||||
if stat.S_ISDIR(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o700:
|
||||
found_dirs.add(relative)
|
||||
elif (
|
||||
stat.S_ISREG(info.st_mode)
|
||||
and info.st_nlink == 1
|
||||
and stat.S_IMODE(info.st_mode) == 0o600
|
||||
):
|
||||
found_files.add(relative)
|
||||
else:
|
||||
raise DeliveryError("private backup special or non-private member")
|
||||
if len(found_files) + len(found_dirs) > MAX_ENTRIES:
|
||||
raise DeliveryError("private backup traversal exceeds bound")
|
||||
if found_dirs != {x["path"] for x in value["directories"]} or found_files != {
|
||||
x["path"] for x in value["files"]
|
||||
}:
|
||||
raise DeliveryError("private backup inventory differs")
|
||||
for item in value["files"]:
|
||||
p = payload / item["path"]
|
||||
if p.stat().st_size != item["bytes"] or digest(p) != item["sha256"]:
|
||||
raise DeliveryError("private backup byte verification failed")
|
||||
return value
|
||||
Reference in New Issue
Block a user