212 lines
8.2 KiB
Python
212 lines
8.2 KiB
Python
"""Finite read-only owner/kernel observation; no execution or admission mutation."""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from . import install, journal, node_health, operator_units, preflight, services
|
|
from .policy import DeliveryError, canonical, digest, integer, read_json
|
|
|
|
NODE = Path("/sys/fs/cgroup/kfonlinejudge.slice")
|
|
UNITS = (*services.PROFILES, "postgres", "rabbitmq", "sandbox-factory", "proxy")
|
|
CPU_FIELDS = {
|
|
"usage_usec",
|
|
"user_usec",
|
|
"system_usec",
|
|
"nr_periods",
|
|
"nr_throttled",
|
|
"throttled_usec",
|
|
}
|
|
MEMORY_FIELDS = {"low", "high", "max", "oom", "oom_kill", "oom_group_kill"}
|
|
IO_FIELDS = {"rbytes", "wbytes", "rios", "wios"}
|
|
|
|
|
|
def text(path: Path, maximum: int = 4096) -> str:
|
|
with path.open() as source:
|
|
value = source.read(maximum + 1)
|
|
if len(value) > maximum:
|
|
raise DeliveryError("kernel observation byte limit")
|
|
return value.strip()
|
|
|
|
|
|
def number(value: str) -> int:
|
|
if not isinstance(value, str) or re.fullmatch(r"[0-9]{1,19}", value) is None:
|
|
raise DeliveryError("invalid kernel counter")
|
|
return integer(int(value), 0, 2**63 - 1, "kernel counter")
|
|
|
|
|
|
def counters(value: str, selected: set[str]) -> dict:
|
|
result = {}
|
|
lines = value.splitlines()
|
|
if not 1 <= len(lines) <= 32:
|
|
raise DeliveryError("kernel counter inventory limit")
|
|
for line in lines:
|
|
parts = line.split()
|
|
if len(parts) != 2 or parts[0] in result:
|
|
raise DeliveryError("invalid kernel counter row")
|
|
result[parts[0]] = number(parts[1])
|
|
if not selected <= result.keys():
|
|
raise DeliveryError("required kernel counter absent")
|
|
return {key: result[key] for key in sorted(selected)}
|
|
|
|
|
|
def io_totals(value: str) -> dict:
|
|
"""Bound the device inventory and omit device identities from the output."""
|
|
lines = value.splitlines()
|
|
if len(lines) > 16:
|
|
raise DeliveryError("kernel IO device inventory limit")
|
|
totals = dict.fromkeys(sorted(IO_FIELDS), 0)
|
|
devices = set()
|
|
for line in lines:
|
|
parts = line.split()
|
|
if not parts or re.fullmatch(r"[0-9]{1,10}:[0-9]{1,10}", parts[0]) is None:
|
|
raise DeliveryError("invalid kernel IO device row")
|
|
if parts[0] in devices or len(parts) > 16:
|
|
raise DeliveryError("duplicate or excessive kernel IO row")
|
|
devices.add(parts[0])
|
|
values = {}
|
|
for member in parts[1:]:
|
|
key, separator, raw = member.partition("=")
|
|
if not separator or key in values:
|
|
raise DeliveryError("invalid kernel IO counter")
|
|
values[key] = number(raw)
|
|
if not IO_FIELDS <= values.keys():
|
|
raise DeliveryError("required kernel IO counter absent")
|
|
for key in totals:
|
|
totals[key] = integer(totals[key] + values[key], 0, 2**63 - 1, "total kernel IO")
|
|
return {"devices_observed": len(devices), **totals}
|
|
|
|
|
|
def kernel(path: Path) -> dict:
|
|
preflight.protected(path, directory=True)
|
|
return {
|
|
"memory_current_bytes": number(text(path / "memory.current")),
|
|
"memory_peak_bytes": number(text(path / "memory.peak")),
|
|
"pids_current": number(text(path / "pids.current")),
|
|
"pids_peak": number(text(path / "pids.peak")),
|
|
"memory_events": counters(text(path / "memory.events"), MEMORY_FIELDS),
|
|
"cpu": counters(text(path / "cpu.stat"), CPU_FIELDS),
|
|
"io": io_totals(text(path / "io.stat", 8192)),
|
|
"limits": {
|
|
key: text(path / key, 64)
|
|
for key in ("memory.max", "memory.swap.max", "pids.max", "cpu.max")
|
|
},
|
|
}
|
|
|
|
|
|
def filesystem(path: Path) -> dict:
|
|
# Aggregate filesystem figures never enumerate source/hidden/credential files.
|
|
if path.is_symlink() or not path.is_dir():
|
|
raise DeliveryError("ordinary observation storage directory required")
|
|
value = os.statvfs(path)
|
|
return {
|
|
"capacity_bytes": value.f_frsize * value.f_blocks,
|
|
"available_bytes": value.f_frsize * value.f_bavail,
|
|
"used_bytes": value.f_frsize * (value.f_blocks - value.f_bfree),
|
|
"inodes_total": value.f_files,
|
|
"inodes_free": value.f_favail,
|
|
"read_only": bool(value.f_flag & os.ST_RDONLY),
|
|
}
|
|
|
|
|
|
def observe() -> dict:
|
|
operator_units.guarded("node-observation")
|
|
selected, _, payload = node_health.binding()
|
|
for name in ("node_observation.py", "operator_units.py"):
|
|
shipped = payload / "delivery/scripts/release" / name
|
|
preflight.protected(shipped)
|
|
if digest(shipped) != digest(Path(__file__).parent / name):
|
|
raise DeliveryError("observation program differs from selected delivery")
|
|
started = time.monotonic()
|
|
result = {
|
|
"schema_version": 1,
|
|
"kind": "kf.node.read-only-observation",
|
|
"source_sha": read_json(payload.parent / "manifest.json")["source_sha"],
|
|
"configuration_id": selected["configuration_id"],
|
|
"observed_unix_ms": time.time_ns() // 1_000_000,
|
|
"read_only": True,
|
|
"isolation_verified": False,
|
|
"complete": False,
|
|
"kernel": {},
|
|
"filesystems": {},
|
|
"owner_health": None,
|
|
"errors": [],
|
|
}
|
|
for label, path in [
|
|
("node", NODE),
|
|
*[(name, NODE / ("kf-" + name + ".service")) for name in UNITS],
|
|
("tasks", NODE / "kf-judge-tasks"),
|
|
]:
|
|
try:
|
|
result["kernel"][label] = kernel(path)
|
|
except (OSError, ValueError):
|
|
result["errors"].append("kernel_" + label + "_unavailable")
|
|
paths = {
|
|
"root": Path("/var/lib/kf-onlinejudge"),
|
|
"artifacts": Path("/var/lib/kf-onlinejudge/artifacts"),
|
|
"imports": Path("/var/lib/kf-onlinejudge/imports"),
|
|
"postgres": Path("/var/lib/kf-onlinejudge/postgres"),
|
|
"rabbitmq": Path("/var/lib/kf-onlinejudge/rabbitmq"),
|
|
"proxy": Path("/var/lib/kf-onlinejudge/proxy"),
|
|
"backups": Path("/var/lib/kf-onlinejudge/backups"),
|
|
}
|
|
try:
|
|
paths["journal"] = Path("/var/log/journal") / (
|
|
journal.machine_id() + "." + journal.NAMESPACE
|
|
)
|
|
except (OSError, ValueError):
|
|
result["errors"].append("filesystem_journal_unavailable")
|
|
for label, path in paths.items():
|
|
try:
|
|
result["filesystems"][label] = filesystem(path)
|
|
except (OSError, ValueError):
|
|
result["errors"].append("filesystem_" + label + "_unavailable")
|
|
try:
|
|
# Owner status/ACL/MQ/factory observers run under their existing fixed
|
|
# identities. Notification probing is off: no publish/ack/claim/cleanup.
|
|
health = operator_units.run("node-health", payload)
|
|
if (
|
|
health.get("source_sha") != result["source_sha"]
|
|
or health.get("configuration_id") != selected["configuration_id"]
|
|
):
|
|
raise DeliveryError("owner health observation binding differs")
|
|
result["owner_health"] = health
|
|
if health.get("errors"):
|
|
result["errors"].append("owner_observation_incomplete")
|
|
except (OSError, ValueError, subprocess.SubprocessError):
|
|
result["errors"].append("owner_observation_unavailable")
|
|
if install.selection(check_configuration=False) != selected:
|
|
result["errors"].append("selected_binding_changed")
|
|
result["sample_duration_ms"] = int((time.monotonic() - started) * 1000)
|
|
result["complete"] = not result["errors"]
|
|
if len(canonical(result)) > 32768:
|
|
raise DeliveryError("read-only observation result exceeds output bound")
|
|
return result
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Finite read-only owner and kernel observation")
|
|
parser.add_argument("--inside", action="store_true", help=argparse.SUPPRESS)
|
|
args = parser.parse_args()
|
|
try:
|
|
if args.inside:
|
|
value = observe()
|
|
else:
|
|
selected = install.selection()
|
|
value = operator_units.run(
|
|
"node-observation", install.RELEASES / selected["release_id"] / "payload"
|
|
)
|
|
print(json.dumps(value, sort_keys=True))
|
|
if not args.inside and not value["complete"]:
|
|
raise SystemExit(2)
|
|
except (OSError, ValueError, subprocess.SubprocessError) as error:
|
|
parser.exit(1, "Read-only node observation unavailable: " + type(error).__name__ + "\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|