feat(delivery): install bounded dependency configuration snapshots
This commit is contained in:
@@ -21,6 +21,7 @@ service_bootstrap = importlib.import_module("scripts.release.service_bootstrap")
|
||||
service_install = importlib.import_module("scripts.release.service_install")
|
||||
factory_service = importlib.import_module("scripts.release.factory_service")
|
||||
dependencies = importlib.import_module("scripts.release.dependencies")
|
||||
dependency_install = importlib.import_module("scripts.release.dependency_install")
|
||||
|
||||
|
||||
def test_delivery_authorities_and_layout():
|
||||
@@ -590,3 +591,99 @@ def test_dependency_templates_keep_socket_auth_network_and_actual_limit_scope():
|
||||
assert "User=root" not in unit and "MemoryMax=512M" in unit
|
||||
assert "MemorySwapMax=0" in unit and "RestartSec=30" in unit
|
||||
assert "RequiresMountsFor=" + dependencies.STORAGE[name]["path"] in unit
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("root", "/shared"),
|
||||
("filesystem", "tmpfs"),
|
||||
("shared_elsewhere", True),
|
||||
("mountpoint", "/var/lib"),
|
||||
("capacity_bytes", 16 * 1024**3),
|
||||
("capacity_bytes", 128 * 1024**2),
|
||||
("capacity_bytes", True),
|
||||
("options", ["rw"]),
|
||||
],
|
||||
)
|
||||
def test_dependency_storage_refuses_soft_unbounded_or_shared_capacity(field, value):
|
||||
observed = {
|
||||
"root": "/",
|
||||
"mountpoint": dependencies.STORAGE["postgres"]["path"],
|
||||
"filesystem": "ext4",
|
||||
"options": ["rw", "nosuid", "nodev", "noexec"],
|
||||
"shared_elsewhere": False,
|
||||
"capacity_bytes": 2 * 1024**3,
|
||||
}
|
||||
dependencies.validate_storage("postgres", observed)
|
||||
observed[field] = value
|
||||
with pytest.raises(policy.DeliveryError, match="hard dependency storage"):
|
||||
dependencies.validate_storage("postgres", observed)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", ["admin", "password", "privilege", "queue", "limit"])
|
||||
def test_dependency_install_refuses_broker_hash_or_authority_tampering(
|
||||
delivery_configuration,
|
||||
mutation,
|
||||
):
|
||||
_, _, _, _, endpoints, _ = delivery_configuration
|
||||
value = dependencies.broker_definitions(endpoints)
|
||||
dependency_install.validate_definitions(value, endpoints)
|
||||
if mutation == "admin":
|
||||
value["users"][0]["tags"] = ["administrator"]
|
||||
elif mutation == "password":
|
||||
value["users"][0]["password_hash"] = dependencies.salted_password("foreign-secret")
|
||||
elif mutation == "privilege":
|
||||
value["permissions"][1]["write"] = ".*"
|
||||
elif mutation == "queue":
|
||||
value["queues"][0]["arguments"]["x-overflow"] = "drop-head"
|
||||
else:
|
||||
value["users"][1]["limits"]["max-connections"] = 1000
|
||||
with pytest.raises(policy.DeliveryError, match="broker"):
|
||||
dependency_install.validate_definitions(value, endpoints)
|
||||
|
||||
|
||||
def test_dependency_receipt_excludes_foreign_files_and_health_claims():
|
||||
receipt = {
|
||||
"schema_version": 1,
|
||||
"source_sha": "123456789abc" + "d" * 28,
|
||||
"configuration_id": "cfg-" + "e" * 24,
|
||||
"configuration_receipt_sha256": "f" * 64,
|
||||
"files": {name: "b" * 64 for name in dependency_install.FILES},
|
||||
"required_storage_mounts": dependencies.STORAGE,
|
||||
"installed": False,
|
||||
"storage_limits_verified": False,
|
||||
"services_started": False,
|
||||
"execution_verified": False,
|
||||
}
|
||||
dependency_install.validate_receipt(receipt)
|
||||
directory = "/etc/kf-dependencies/sd-123456789abc.cfg-" + "e" * 24
|
||||
assert str(dependency_install.batch_directory(receipt)) == directory
|
||||
assert (
|
||||
str(
|
||||
dependency_install.previous_batch(
|
||||
directory + "/kf-postgres.service", "kf-postgres.service"
|
||||
)
|
||||
)
|
||||
== directory
|
||||
)
|
||||
receipt["execution_verified"] = True
|
||||
with pytest.raises(policy.DeliveryError, match="binding"):
|
||||
dependency_install.validate_receipt(receipt)
|
||||
receipt["execution_verified"] = False
|
||||
receipt["files"]["server.key"] = "b" * 64
|
||||
with pytest.raises(policy.DeliveryError, match="inventory"):
|
||||
dependency_install.validate_receipt(receipt)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target",
|
||||
[
|
||||
"/etc/kf-onlinejudge/units/sd-123456789abc.cfg-" + "e" * 24 + "/kf-postgres.service",
|
||||
"/etc/kf-dependencies/sd-123456789abc.cfg-" + "e" * 24 + "/../kf-postgres.service",
|
||||
"/etc//kf-dependencies/sd-123456789abc.cfg-" + "e" * 24 + "/kf-postgres.service",
|
||||
],
|
||||
)
|
||||
def test_dependency_pointer_does_not_adopt_unknown_units_or_paths(target):
|
||||
with pytest.raises(policy.DeliveryError, match="unknown dependency unit"):
|
||||
dependency_install.previous_batch(target, "kf-postgres.service")
|
||||
|
||||
@@ -201,3 +201,25 @@ Dependency directive references: [PostgreSQL resources](https://www.postgresql.o
|
||||
[RabbitMQ limits](https://www.rabbitmq.com/docs/limits),
|
||||
[definition import](https://www.rabbitmq.com/docs/definitions),
|
||||
and [RabbitMQ access control](https://www.rabbitmq.com/docs/access-control).
|
||||
|
||||
`python3.13 -m scripts.release.dependency_install DIRECTORY --receipt-sha256 SHA
|
||||
--erlang-cookie PRIVATE_FILE --cookie-sha256 SHA` installs the selected dependency
|
||||
snapshot with all app/dependency services stopped. It requires already provisioned
|
||||
dedicated ext4 mounts at the two fixed data paths, with their fixed capacity and
|
||||
`rw,nosuid,nodev,noexec` options. It does not create, format, resize or mount disks.
|
||||
A root-owned empty mount (apart from an empty private `lost+found`) may be assigned
|
||||
to its dedicated account; existing data is never recursively reassigned. Ordinary
|
||||
UID/GID 1203/1204 must be locked, without supplementary authority, and have the
|
||||
fixed PostgreSQL/RabbitMQ service homes. Conflicting OS accounts are not adopted.
|
||||
|
||||
The generated files are installed under `/etc/kf-dependencies/RELEASE.CONFIGURATION/`.
|
||||
This separate root-owned searchable tree lets each service group read only its
|
||||
0640 configuration files; the application configuration tree remains private.
|
||||
The definition file's salted credentials, user privileges and queue budgets must
|
||||
match the selected outbox/Worker batch. The operator supplies an explicit private
|
||||
256-bit Erlang cookie; repeat installation requires the same ordinary-owned 0400
|
||||
cookie and does not implicitly rotate it. No cookie/password bytes are reported.
|
||||
System unit pointers may replace only recorded root-owned dependency snapshots.
|
||||
The command retains old batches, verifies loaded/inactive metadata, starts no
|
||||
service and initializes no database. Cluster initialization, actual effective
|
||||
limits, broker authorization and startup/readiness still require VM acceptance.
|
||||
|
||||
@@ -23,9 +23,56 @@ QUEUE_ARGUMENTS = {
|
||||
"x-message-ttl": 600000,
|
||||
}
|
||||
STORAGE = {
|
||||
"postgres": {"path": "/var/lib/kf-onlinejudge/postgres", "filesystem_bytes": 2 * 1024**3},
|
||||
"rabbitmq": {"path": "/var/lib/kf-onlinejudge/rabbitmq", "filesystem_bytes": 512 * 1024**2},
|
||||
"postgres": {
|
||||
"path": "/var/lib/kf-onlinejudge/postgres",
|
||||
"filesystem_bytes": 2 * 1024**3,
|
||||
},
|
||||
"rabbitmq": {
|
||||
"path": "/var/lib/kf-onlinejudge/rabbitmq",
|
||||
"filesystem_bytes": 512 * 1024**2,
|
||||
},
|
||||
}
|
||||
CONFIGURATION_ROOT = Path("/etc/kf-dependencies")
|
||||
|
||||
|
||||
def storage_observation(name: str) -> dict:
|
||||
"""Read an already provisioned dedicated filesystem, never create/mount one."""
|
||||
directory = Path(STORAGE[name]["path"])
|
||||
preflight.protected(directory.parent, directory=True)
|
||||
if directory.is_symlink() or not directory.is_dir():
|
||||
raise DeliveryError("ordinary dependency data directory required")
|
||||
mounts = [line.split() for line in Path("/proc/self/mountinfo").read_text().splitlines()]
|
||||
rows = [row for row in mounts if len(row) > 6 and row[4] == str(directory)]
|
||||
if len(rows) != 1:
|
||||
raise DeliveryError("one dedicated dependency storage mount required")
|
||||
row = rows[0]
|
||||
separator = row.index("-")
|
||||
fs = os.statvfs(directory)
|
||||
observation = {
|
||||
"root": row[3],
|
||||
"mountpoint": row[4],
|
||||
"filesystem": row[separator + 1],
|
||||
"options": row[5].split(","),
|
||||
"device": row[2],
|
||||
"shared_elsewhere": any(other[2] == row[2] and other != row for other in mounts),
|
||||
"capacity_bytes": fs.f_frsize * fs.f_blocks,
|
||||
}
|
||||
validate_storage(name, observation)
|
||||
return observation
|
||||
|
||||
|
||||
def validate_storage(name: str, observation: dict) -> None:
|
||||
required = STORAGE[name]["filesystem_bytes"]
|
||||
if (
|
||||
observation["root"] != "/"
|
||||
or observation["mountpoint"] != STORAGE[name]["path"]
|
||||
or observation["filesystem"] != "ext4"
|
||||
or observation["shared_elsewhere"] is not False
|
||||
or not {"rw", "nosuid", "nodev", "noexec"} <= set(observation["options"])
|
||||
or type(observation["capacity_bytes"]) is not int
|
||||
or not required * 9 // 10 <= observation["capacity_bytes"] <= required
|
||||
):
|
||||
raise DeliveryError("dedicated hard dependency storage budget differs")
|
||||
|
||||
|
||||
def check_runtime(name: str) -> dict:
|
||||
@@ -89,20 +136,11 @@ def check_runtime(name: str) -> dict:
|
||||
or stat.S_IMODE(info.st_mode) != 0o700
|
||||
):
|
||||
raise DeliveryError("private dependency data identity/mode differs")
|
||||
# A parent filesystem and soft DB/broker alarm must not pass as the
|
||||
# separately bounded data filesystem required by the service profile.
|
||||
mounts = [line.split() for line in Path("/proc/self/mountinfo").read_text().splitlines()]
|
||||
if not any(len(row) > 6 and row[4] == str(directory) and row[3] == "/" for row in mounts):
|
||||
raise DeliveryError("separate dependency storage mount required")
|
||||
fs = os.statvfs(directory)
|
||||
capacity = fs.f_frsize * fs.f_blocks
|
||||
required = STORAGE[name]["filesystem_bytes"]
|
||||
if not required * 9 // 10 <= capacity <= required:
|
||||
raise DeliveryError("effective dependency filesystem capacity differs")
|
||||
storage = storage_observation(name)
|
||||
return {
|
||||
"dependency": name,
|
||||
"actual_cgroup_limits": limits,
|
||||
"data_filesystem_bytes": capacity,
|
||||
"data_filesystem_bytes": storage["capacity_bytes"],
|
||||
"execution_verified": False,
|
||||
}
|
||||
|
||||
@@ -152,7 +190,7 @@ def broker_definitions(connections: dict) -> dict:
|
||||
"vhost": "kf",
|
||||
"durable": True,
|
||||
"auto_delete": False,
|
||||
"arguments": QUEUE_ARGUMENTS,
|
||||
"arguments": dict(QUEUE_ARGUMENTS),
|
||||
}
|
||||
],
|
||||
}
|
||||
@@ -166,7 +204,9 @@ def render(release: str, batch: str) -> dict[str, bytes]:
|
||||
r"cfg-[0-9a-f]{24}", batch
|
||||
):
|
||||
raise DeliveryError("invalid dependency configuration binding")
|
||||
base = f"/etc/kf-onlinejudge/dependencies/{release}.{batch}"
|
||||
# The application's /etc/kf-onlinejudge is 0700: dependency identities
|
||||
# must use a separate root-owned searchable tree with group-readable files.
|
||||
base = str(CONFIGURATION_ROOT / f"{release}.{batch}")
|
||||
pg = (
|
||||
"\n".join(
|
||||
[
|
||||
@@ -304,21 +344,26 @@ def render(release: str, batch: str) -> dict[str, bytes]:
|
||||
values += [
|
||||
"RuntimeDirectory=postgresql",
|
||||
"RuntimeDirectoryMode=0755",
|
||||
(
|
||||
"ExecStart=/usr/bin/env -i PATH=/usr/bin:/bin LC_ALL=C.UTF-8"
|
||||
f" /usr/bin/postgres -c config_file={base}/postgresql.conf",
|
||||
f" /usr/bin/postgres -c config_file={base}/postgresql.conf"
|
||||
),
|
||||
]
|
||||
else:
|
||||
values += [
|
||||
"WorkingDirectory=/var/lib/kf-onlinejudge/rabbitmq",
|
||||
(
|
||||
"ExecStart=/usr/bin/env -i PATH=/usr/bin:/bin LC_ALL=C.UTF-8"
|
||||
" HOME=/var/lib/kf-onlinejudge/rabbitmq ERL_EPMD_ADDRESS=127.0.0.1"
|
||||
" RABBITMQ_NODENAME=kf@localhost RABBITMQ_DIST_PORT=25672"
|
||||
f" RABBITMQ_CONFIG_FILE={base}/rabbitmq.conf"
|
||||
f" RABBITMQ_ENABLED_PLUGINS_FILE={base}/enabled_plugins"
|
||||
" RABBITMQ_MNESIA_BASE=/var/lib/kf-onlinejudge/rabbitmq/data"
|
||||
' RABBITMQ_LOGS=- "RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=+S 2:2 +A 4 +P 1024 +Q 1024'
|
||||
' RABBITMQ_LOGS=- "RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS='
|
||||
"+S 2:2 +A 4 +P 1024 +Q 1024"
|
||||
' -kernel inet_dist_use_interface {127,0,0,1}"'
|
||||
" /usr/lib/rabbitmq/sbin/rabbitmq-server",
|
||||
" /usr/lib/rabbitmq/sbin/rabbitmq-server"
|
||||
),
|
||||
]
|
||||
values += ["[Install]", "WantedBy=multi-user.target"]
|
||||
files["kf-" + name + ".service"] = ("\n".join(values) + "\n").encode()
|
||||
@@ -377,6 +422,7 @@ def generate(connections_path: Path, output: Path) -> dict:
|
||||
"schema_version": 1,
|
||||
"source_sha": original["source_sha"],
|
||||
"configuration_id": selected["configuration_id"],
|
||||
"configuration_receipt_sha256": selected["receipt_sha256"],
|
||||
"files": hashes,
|
||||
"required_storage_mounts": STORAGE,
|
||||
"installed": False,
|
||||
|
||||
@@ -0,0 +1,535 @@
|
||||
"""Root dependency snapshots on pre-provisioned storage; never initialize/start DBs."""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import grp
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import pwd
|
||||
import re
|
||||
import secrets
|
||||
import stat
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from . import configuration, dependencies, install, preflight, service_install
|
||||
from .policy import DeliveryError, canonical, digest, fields, read_json, sha256
|
||||
|
||||
ROOT = dependencies.CONFIGURATION_ROOT
|
||||
NAMES = {"kf-postgres.service", "kf-rabbitmq.service"}
|
||||
FILES = {
|
||||
"postgresql.conf": (0o640, 1203),
|
||||
"pg_hba.conf": (0o640, 1203),
|
||||
"rabbitmq.conf": (0o640, 1204),
|
||||
"enabled_plugins": (0o640, 1204),
|
||||
"definitions.json": (0o640, 1204),
|
||||
**{name: (0o644, 0) for name in NAMES},
|
||||
}
|
||||
IDENTITIES = {
|
||||
"postgres": (1203, "/var/lib/postgres"),
|
||||
"rabbitmq": (1204, "/var/lib/rabbitmq"),
|
||||
}
|
||||
|
||||
|
||||
def validate_receipt(value: dict) -> dict:
|
||||
fields(
|
||||
value,
|
||||
{
|
||||
"schema_version",
|
||||
"source_sha",
|
||||
"configuration_id",
|
||||
"configuration_receipt_sha256",
|
||||
"files",
|
||||
"required_storage_mounts",
|
||||
"installed",
|
||||
"storage_limits_verified",
|
||||
"services_started",
|
||||
"execution_verified",
|
||||
},
|
||||
"dependency receipt",
|
||||
)
|
||||
if (
|
||||
type(value["schema_version"]) is not int
|
||||
or value["schema_version"] != 1
|
||||
or not isinstance(value["source_sha"], str)
|
||||
or not re.fullmatch(r"[0-9a-f]{40}", value["source_sha"])
|
||||
or not isinstance(value["configuration_id"], str)
|
||||
or not re.fullmatch(r"cfg-[0-9a-f]{24}", value["configuration_id"])
|
||||
or value["required_storage_mounts"] != dependencies.STORAGE
|
||||
or any(
|
||||
value[key] is not False
|
||||
for key in (
|
||||
"installed",
|
||||
"storage_limits_verified",
|
||||
"services_started",
|
||||
"execution_verified",
|
||||
)
|
||||
)
|
||||
):
|
||||
raise DeliveryError("dependency receipt binding differs")
|
||||
sha256(value["configuration_receipt_sha256"])
|
||||
fields(value["files"], set(FILES), "dependency file inventory")
|
||||
for checksum in value["files"].values():
|
||||
sha256(checksum)
|
||||
return value
|
||||
|
||||
|
||||
def batch_directory(receipt: dict) -> Path:
|
||||
validate_receipt(receipt)
|
||||
return ROOT / ("sd-" + receipt["source_sha"][:12] + "." + receipt["configuration_id"])
|
||||
|
||||
|
||||
def previous_batch(target: str, name: str) -> Path:
|
||||
path = Path(target)
|
||||
if (
|
||||
name not in NAMES
|
||||
or path.name != name
|
||||
or path.parent.parent != ROOT
|
||||
or not re.fullmatch(r"sd-[0-9a-f]{12}\.cfg-[0-9a-f]{24}", path.parent.name)
|
||||
or path.as_posix() != target
|
||||
):
|
||||
raise DeliveryError("unknown dependency unit target; operator review required")
|
||||
return path.parent
|
||||
|
||||
|
||||
def check_group(directory: Path, receipt: dict) -> None:
|
||||
preflight.protected(directory, directory=True)
|
||||
if stat.S_IMODE(directory.stat().st_mode) != 0o711:
|
||||
raise DeliveryError("protected searchable dependency batch required")
|
||||
if {p.name for p in directory.iterdir()} != set(FILES) | {"receipt.json"}:
|
||||
raise DeliveryError("unexpected installed dependency member")
|
||||
path = directory / "receipt.json"
|
||||
preflight.protected(path)
|
||||
configuration.private(path)
|
||||
if canonical(validate_receipt(read_json(path, 16384))) != canonical(receipt):
|
||||
raise DeliveryError("installed dependency receipt differs")
|
||||
for name, (mode, group) in FILES.items():
|
||||
path = directory / name
|
||||
preflight.protected(path)
|
||||
info = path.stat()
|
||||
if (
|
||||
info.st_size > 65536
|
||||
or stat.S_IMODE(info.st_mode) != mode
|
||||
or info.st_gid != group
|
||||
or digest(path) != receipt["files"][name]
|
||||
):
|
||||
raise DeliveryError("installed dependency bytes/permissions differ")
|
||||
|
||||
|
||||
def known_link(path: Path) -> Path | None:
|
||||
try:
|
||||
info = path.lstat()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if not stat.S_ISLNK(info.st_mode) or info.st_uid != 0 or info.st_gid != 0:
|
||||
raise DeliveryError("unknown dependency system unit; refused overwrite")
|
||||
directory = previous_batch(os.readlink(path), path.name)
|
||||
receipt_path = directory / "receipt.json"
|
||||
preflight.protected(receipt_path)
|
||||
configuration.private(receipt_path)
|
||||
receipt = validate_receipt(read_json(receipt_path, 16384))
|
||||
if batch_directory(receipt) != directory:
|
||||
raise DeliveryError("old dependency unit binding differs")
|
||||
check_group(directory, receipt)
|
||||
return directory
|
||||
|
||||
|
||||
def check_accounts() -> None:
|
||||
for name, (identity, home) in IDENTITIES.items():
|
||||
try:
|
||||
if pwd.getpwuid(identity).pw_name != name:
|
||||
raise DeliveryError("dependency uid already assigned")
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
if grp.getgrgid(identity).gr_name != name:
|
||||
raise DeliveryError("dependency gid already assigned")
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
account = pwd.getpwnam(name)
|
||||
if (account.pw_uid, account.pw_gid, account.pw_dir, account.pw_shell) != (
|
||||
identity,
|
||||
identity,
|
||||
home,
|
||||
"/usr/bin/nologin",
|
||||
):
|
||||
raise DeliveryError("existing ordinary dependency identity differs")
|
||||
preflight.protected(Path("/etc/shadow"))
|
||||
shadow = Path("/etc/shadow")
|
||||
if shadow.stat().st_size > 1024 * 1024:
|
||||
raise DeliveryError("shadow identity observation exceeds bound")
|
||||
rows = [line.split(":") for line in shadow.read_text().splitlines()]
|
||||
passwords = [row[1] for row in rows if len(row) == 9 and row[0] == name]
|
||||
if len(passwords) != 1 or not passwords[0].startswith(("!", "*")):
|
||||
raise DeliveryError("dependency login identity must be locked")
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
group = grp.getgrnam(name)
|
||||
if group.gr_gid != identity or group.gr_mem:
|
||||
raise DeliveryError("dependency group authority differs")
|
||||
except KeyError:
|
||||
pass
|
||||
if any(name in group.gr_mem for group in grp.getgrall()):
|
||||
raise DeliveryError("dependency supplementary authority refused")
|
||||
|
||||
|
||||
def ensure_accounts() -> None:
|
||||
check_accounts()
|
||||
for name, (identity, home) in IDENTITIES.items():
|
||||
try:
|
||||
grp.getgrnam(name)
|
||||
except KeyError:
|
||||
subprocess.run(
|
||||
["/usr/bin/groupadd", "--gid", str(identity), name],
|
||||
check=True,
|
||||
timeout=10,
|
||||
capture_output=True,
|
||||
env={"PATH": "/usr/bin:/bin", "LC_ALL": "C"},
|
||||
)
|
||||
try:
|
||||
pwd.getpwnam(name)
|
||||
except KeyError:
|
||||
subprocess.run(
|
||||
[
|
||||
"/usr/bin/useradd",
|
||||
"--uid",
|
||||
str(identity),
|
||||
"--gid",
|
||||
str(identity),
|
||||
"--no-create-home",
|
||||
"--no-user-group",
|
||||
"--key",
|
||||
"SUB_UID_COUNT=0",
|
||||
"--key",
|
||||
"SUB_GID_COUNT=0",
|
||||
"--home-dir",
|
||||
home,
|
||||
"--shell",
|
||||
"/usr/bin/nologin",
|
||||
name,
|
||||
],
|
||||
check=True,
|
||||
timeout=10,
|
||||
capture_output=True,
|
||||
env={"PATH": "/usr/bin:/bin", "LC_ALL": "C"},
|
||||
)
|
||||
check_accounts()
|
||||
for name in IDENTITIES:
|
||||
pwd.getpwnam(name)
|
||||
grp.getgrnam(name)
|
||||
|
||||
|
||||
def require_stopped() -> None:
|
||||
install.require_stopped()
|
||||
for name in sorted(NAMES):
|
||||
values = dict(
|
||||
line.split("=", 1)
|
||||
for line in service_install.systemctl(
|
||||
"show", name, "--property=LoadState,ActiveState,MainPID"
|
||||
).splitlines()
|
||||
if "=" in line
|
||||
)
|
||||
if (
|
||||
values.get("LoadState") not in {"loaded", "not-found"}
|
||||
or values.get("ActiveState") != "inactive"
|
||||
or values.get("MainPID") != "0"
|
||||
):
|
||||
raise DeliveryError("all dependency units must be stopped")
|
||||
for port in (5432, 5672, 25672, 4369):
|
||||
import socket
|
||||
|
||||
with socket.socket() as check:
|
||||
try:
|
||||
check.bind(("127.0.0.1", port))
|
||||
except OSError as error:
|
||||
raise DeliveryError("dependency port belongs to a running process") from error
|
||||
if Path("/run/postgresql/.s.PGSQL.5432").exists():
|
||||
raise DeliveryError("dependency DB socket remains")
|
||||
|
||||
|
||||
def validate_definitions(value: dict, endpoints: dict) -> None:
|
||||
"""Match exact broker privileges and independently check salted credentials."""
|
||||
expected = dependencies.broker_definitions(endpoints)
|
||||
fields(value, set(expected), "broker definitions")
|
||||
if not isinstance(value["users"], list) or len(value["users"]) != 2:
|
||||
raise DeliveryError("broker user inventory differs")
|
||||
for actual, wanted in zip(value["users"], expected["users"], strict=True):
|
||||
fields(actual, set(wanted), "broker user")
|
||||
encoded = actual["password_hash"]
|
||||
if not isinstance(encoded, str) or len(encoded) != 48:
|
||||
raise DeliveryError("broker password fingerprint encoding differs")
|
||||
try:
|
||||
packed = base64.b64decode(encoded, validate=True)
|
||||
except ValueError as error:
|
||||
raise DeliveryError("broker password fingerprint encoding differs") from error
|
||||
password = urlsplit(endpoints["amqp"][wanted["name"].removeprefix("kf_")]).password
|
||||
if len(packed) != 36 or not hmac.compare_digest(
|
||||
packed[4:], hashlib.sha256(packed[:4] + password.encode()).digest()
|
||||
):
|
||||
raise DeliveryError("broker password differs from selected batch")
|
||||
wanted["password_hash"] = encoded
|
||||
if canonical(value) != canonical(expected):
|
||||
raise DeliveryError("broker privileges, queue or budget differ")
|
||||
|
||||
|
||||
def installed_endpoints(selected: dict) -> dict:
|
||||
def env(name: str) -> dict:
|
||||
return configuration.environment_read(
|
||||
configuration.installed_path(
|
||||
selected["release_id"], selected["configuration_id"], name
|
||||
).read_bytes()
|
||||
)
|
||||
|
||||
def cfg(name: str) -> dict:
|
||||
return read_json(
|
||||
configuration.installed_path(
|
||||
selected["release_id"], selected["configuration_id"], name
|
||||
),
|
||||
32768,
|
||||
)
|
||||
|
||||
def db(name: str, key: str = "KF_OJ_DATABASE_URL") -> str:
|
||||
return env(name)[key].replace("postgresql+asyncpg:", "postgresql:", 1)
|
||||
|
||||
return configuration.validate_connections(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"database": {
|
||||
"api": db("api.env"),
|
||||
"coordinator": cfg("coordinator.json")["database_url"],
|
||||
"import": db("import.env"),
|
||||
"migration": db("migration.env"),
|
||||
"identity": db("identity.env", "KF_OJ_IDENTITY_OPERATOR_DATABASE_URL"),
|
||||
},
|
||||
"amqp": {
|
||||
"outbox": env("outbox.env")["KF_OJ_AMQP_URL"],
|
||||
"worker": cfg("worker.json")["amqp_url"],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def write_group_file(path: Path, contents: bytes, mode: int, group: int) -> None:
|
||||
# Publish only root-owned immutable bytes, with one dedicated read group.
|
||||
if path.exists() or path.is_symlink():
|
||||
preflight.protected(path)
|
||||
info = path.stat()
|
||||
if (stat.S_IMODE(info.st_mode), info.st_gid) != (
|
||||
mode,
|
||||
group,
|
||||
) or path.read_bytes() != contents:
|
||||
raise DeliveryError("existing dependency file differs; refused overwrite")
|
||||
return
|
||||
install.exclusive_file(path, contents, 0o600)
|
||||
os.chown(path, 0, group)
|
||||
path.chmod(mode)
|
||||
install.sync_directory(path.parent)
|
||||
|
||||
|
||||
def storage_ownership(name: str) -> None:
|
||||
path = Path(dependencies.STORAGE[name]["path"])
|
||||
identity = IDENTITIES[name][0]
|
||||
info = path.lstat()
|
||||
if (info.st_uid, info.st_gid, stat.S_IMODE(info.st_mode)) == (
|
||||
identity,
|
||||
identity,
|
||||
0o700,
|
||||
):
|
||||
return
|
||||
if (info.st_uid, info.st_gid, stat.S_IMODE(info.st_mode)) != (0, 0, 0o700):
|
||||
raise DeliveryError("dependency mount ownership conflicts")
|
||||
# An empty, deliberately provisioned ext4 volume can contain only its
|
||||
# private, empty recovery directory. Never recursively chown existing data.
|
||||
for member in path.iterdir():
|
||||
if member.name != "lost+found":
|
||||
raise DeliveryError("unrecognized data on root-owned dependency mount")
|
||||
preflight.protected(member, directory=True)
|
||||
if stat.S_IMODE(member.stat().st_mode) != 0o700 or next(member.iterdir(), None) is not None:
|
||||
raise DeliveryError("dependency recovery directory is not empty/private")
|
||||
os.chown(path, identity, identity)
|
||||
|
||||
|
||||
def install_cookie(source: Path, expected: str) -> None:
|
||||
preflight.protected(source)
|
||||
configuration.private(source)
|
||||
if source.stat().st_size != 65 or digest(source) != sha256(expected):
|
||||
raise DeliveryError("explicit private broker cookie fingerprint differs")
|
||||
contents = source.read_bytes()
|
||||
if re.fullmatch(rb"[0-9a-f]{64}\n", contents) is None:
|
||||
raise DeliveryError("broker cookie must be an explicit 256-bit hexadecimal value")
|
||||
path = Path(dependencies.STORAGE["rabbitmq"]["path"]) / ".erlang.cookie"
|
||||
if path.exists() or path.is_symlink():
|
||||
info = path.lstat()
|
||||
if (
|
||||
not stat.S_ISREG(info.st_mode)
|
||||
or info.st_nlink != 1
|
||||
or (info.st_uid, info.st_gid, stat.S_IMODE(info.st_mode)) != (1204, 1204, 0o400)
|
||||
or info.st_size != 65
|
||||
or path.read_bytes() != contents
|
||||
):
|
||||
raise DeliveryError("existing broker cookie differs; refused rotation")
|
||||
return
|
||||
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_WRONLY, 0o600)
|
||||
try:
|
||||
with os.fdopen(fd, "wb", closefd=False) as output:
|
||||
output.write(contents)
|
||||
output.flush()
|
||||
os.fsync(fd)
|
||||
os.fchown(fd, 1204, 1204)
|
||||
os.fchmod(fd, 0o400)
|
||||
finally:
|
||||
os.close(fd)
|
||||
install.sync_directory(path.parent)
|
||||
|
||||
|
||||
def install_snapshot(source: Path, expected: str, cookie: Path, cookie_sha: str) -> dict:
|
||||
if os.getuid() != 0 or os.geteuid() != 0:
|
||||
raise DeliveryError("dependency installation requires root operator")
|
||||
preflight.protected(source, directory=True)
|
||||
path = source / "receipt.json"
|
||||
preflight.protected(path)
|
||||
configuration.private(path)
|
||||
if digest(path) != sha256(expected):
|
||||
raise DeliveryError("dependency receipt fingerprint differs")
|
||||
receipt = validate_receipt(read_json(path, 16384))
|
||||
selected, original = service_install.current_receipt()
|
||||
if (
|
||||
receipt["source_sha"] != original["source_sha"]
|
||||
or receipt["configuration_id"] != selected["configuration_id"]
|
||||
or receipt["configuration_receipt_sha256"] != selected["receipt_sha256"]
|
||||
):
|
||||
raise DeliveryError("dependency snapshot is not the selected binding")
|
||||
payload = install.RELEASES / selected["release_id"] / "payload"
|
||||
for name in ("dependencies.py", "dependency_install.py"):
|
||||
shipped = payload / "delivery/scripts/release" / name
|
||||
preflight.protected(shipped)
|
||||
if digest(shipped) != digest(Path(__file__).parent / name):
|
||||
raise DeliveryError("dependency installer differs from selected delivery")
|
||||
if {p.name for p in source.iterdir()} != set(FILES) | {"receipt.json"}:
|
||||
raise DeliveryError("unexpected dependency source member")
|
||||
contents = dependencies.render(selected["release_id"], selected["configuration_id"])
|
||||
for name in FILES:
|
||||
path = source / name
|
||||
preflight.protected(path)
|
||||
configuration.private(path)
|
||||
if path.stat().st_size > 65536 or digest(path) != receipt["files"][name]:
|
||||
raise DeliveryError("dependency source fingerprint differs")
|
||||
if name != "definitions.json" and path.read_bytes() != contents[name]:
|
||||
raise DeliveryError("dependency file is not the fixed trusted composition")
|
||||
validate_definitions(
|
||||
read_json(source / "definitions.json", 65536), installed_endpoints(selected)
|
||||
)
|
||||
contents["definitions.json"] = (source / "definitions.json").read_bytes()
|
||||
require_stopped()
|
||||
check_accounts()
|
||||
storage = {name: dependencies.storage_observation(name) for name in IDENTITIES}
|
||||
preflight.protected(service_install.SYSTEM_UNITS, directory=True)
|
||||
for name in NAMES:
|
||||
known_link(service_install.SYSTEM_UNITS / name)
|
||||
ensure_accounts()
|
||||
for name in IDENTITIES:
|
||||
storage_ownership(name)
|
||||
install_cookie(cookie, cookie_sha)
|
||||
install.directory(ROOT, 0o711)
|
||||
directory = batch_directory(receipt)
|
||||
install.directory(directory, 0o711)
|
||||
if not {p.name for p in directory.iterdir()} <= set(FILES) | {"receipt.json"}:
|
||||
raise DeliveryError("unexpected partial dependency batch member")
|
||||
for member in directory.iterdir():
|
||||
preflight.protected(member)
|
||||
if member.stat().st_size > 65536:
|
||||
raise DeliveryError("partial dependency member exceeds bound")
|
||||
for name, (mode, group) in FILES.items():
|
||||
write_group_file(directory / name, contents[name], mode, group)
|
||||
install.exclusive_file(directory / "receipt.json", canonical(receipt), 0o600)
|
||||
check_group(directory, receipt)
|
||||
if install.selection() != selected:
|
||||
raise DeliveryError("selected binding changed during dependency installation")
|
||||
require_stopped()
|
||||
for name in sorted(NAMES):
|
||||
path = service_install.SYSTEM_UNITS / name
|
||||
if known_link(path) == directory:
|
||||
continue
|
||||
temporary = path.parent / ("." + name + ".dependency-" + secrets.token_hex(8))
|
||||
created = None
|
||||
try:
|
||||
temporary.symlink_to(directory / name)
|
||||
info = temporary.lstat()
|
||||
created = (info.st_dev, info.st_ino)
|
||||
known_link(path)
|
||||
os.replace(temporary, path)
|
||||
install.sync_directory(path.parent)
|
||||
finally:
|
||||
if created is not None and temporary.is_symlink():
|
||||
info = temporary.lstat()
|
||||
if (info.st_dev, info.st_ino) == created:
|
||||
temporary.unlink()
|
||||
service_install.systemctl("daemon-reload")
|
||||
loaded = {}
|
||||
for name in sorted(NAMES):
|
||||
values = dict(
|
||||
line.split("=", 1)
|
||||
for line in service_install.systemctl(
|
||||
"show", name, "--property=LoadState,ActiveState,MainPID,FragmentPath"
|
||||
).splitlines()
|
||||
if "=" in line
|
||||
)
|
||||
if (
|
||||
values.get("LoadState") != "loaded"
|
||||
or values.get("ActiveState") != "inactive"
|
||||
or values.get("MainPID") != "0"
|
||||
or values.get("FragmentPath")
|
||||
not in {str(service_install.SYSTEM_UNITS / name), str(directory / name)}
|
||||
or known_link(service_install.SYSTEM_UNITS / name) != directory
|
||||
):
|
||||
raise DeliveryError("dependency unit load readback differs")
|
||||
loaded[name] = values
|
||||
if install.selection() != selected:
|
||||
raise DeliveryError("selected binding changed during dependency loading")
|
||||
return {
|
||||
"source_sha": receipt["source_sha"],
|
||||
"configuration_id": receipt["configuration_id"],
|
||||
"units_installed": True,
|
||||
"storage_mount_readback": storage,
|
||||
"unit_manager_readback": loaded,
|
||||
"cluster_initialized": False,
|
||||
"services_started": False,
|
||||
"execution_verified": False,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Protected dependency snapshot installation")
|
||||
parser.add_argument("directory", type=Path)
|
||||
parser.add_argument("--receipt-sha256", required=True)
|
||||
parser.add_argument("--erlang-cookie", required=True, type=Path)
|
||||
parser.add_argument("--cookie-sha256", required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
print(
|
||||
json.dumps(
|
||||
install_snapshot(
|
||||
args.directory,
|
||||
args.receipt_sha256,
|
||||
args.erlang_cookie,
|
||||
args.cookie_sha256,
|
||||
),
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
except (OSError, ValueError, subprocess.SubprocessError) as error:
|
||||
parser.exit(
|
||||
1,
|
||||
"Dependency installation refused: "
|
||||
+ (str(error) if isinstance(error, DeliveryError) else type(error).__name__)
|
||||
+ "\n",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user