Kronecker/KF-OnlineJudge
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(delivery): fit bounded health probes into startup budgets

Browse Source
This commit is contained in:
Kronecker
2026-10-15 21:50:57 +08:00
parent 04e5fac311
commit 721c293382
6 changed files with 251 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/release/README.md
+10 -3
View File
@@ -415,9 +415,16 @@ end-to-end target still requires measured acceptance.
Fixed health helpers include interpreter startup in their wall-clock deadline.
Outbox and import queries have 60 seconds each; factory status has 30 seconds and
broker metadata has 45 seconds. Node readiness has 300 seconds for the sequential
checks. Each helper keeps its fixed CPU, memory, process and privilege limits.
These operator deadlines do not change student execution budgets. An expired or
failed query keeps admission closed.
checks. Each ordinary helper has one CPU within the node's two-CPU ceiling;
its memory, process and privilege limits remain fixed. These operator budgets
do not change student execution budgets. Startup opens admission only after all
required readiness queries succeed.
Broker metadata uses one fixed read-only RPC with a five-second remote deadline
and a 30-second CLI process deadline, within the helper's 45-second limit. It reads
queue metadata, vhost limits, usernames/tags and permissions; it does not read
messages or password hashes. Readiness requires exactly the configured classic
queue and budgets, including RabbitMQ's optional `x-queue-type=classic` argument.
The broker helper has a 64 MiB per-file ceiling for Erlang JIT mappings.
[OTP 27.3.4](https://github.com/erlang/otp/blob/OTP-27.3.4/erts/emulator/beam/jit/beam_jit_main.cpp)
scripts/release/broker_health.py
+9 -76
View File
@@ -4,56 +4,33 @@ import argparse
import json
import stat
import subprocess
import tempfile
from pathlib import Path
from . import broker_snapshot, operator_units, preflight
from . import dependency_policy as dependencies
from . import operator_units, preflight
from .policy import DeliveryError, canonical, fields, integer
CTL = Path("/usr/lib/rabbitmq/sbin/rabbitmqctl")
QUERIES = {
"queues": [
"list_queues",
"-p",
"kf",
"name",
"durable",
"auto_delete",
"exclusive",
"type",
"state",
"arguments",
"messages_ready",
"messages_unacknowledged",
"message_bytes",
"consumers",
"policy",
"operator_policy",
"effective_policy_definition",
],
"vhost_limits": ["list_vhost_limits", "--global"],
"users": ["list_users"],
"permissions": ["list_permissions", "-p", "kf"],
}
CTL = broker_snapshot.CTL
def queue_arguments(value) -> dict:
if isinstance(value, dict):
result = value
elif isinstance(value, list) and len(value) == 4:
result = dict(value)
elif isinstance(value, list) and len(value) in (4, 5):
result = {}
for row in value:
if not isinstance(row, list) or len(row) != 3 or row[0] in result:
raise DeliveryError("duplicate/unknown broker argument")
name, kind, argument = row
if name not in dependencies.QUEUE_ARGUMENTS or kind != (
"longstr" if name == "x-overflow" else "long"
if name not in {*dependencies.QUEUE_ARGUMENTS, "x-queue-type"} or kind != (
"longstr" if name in {"x-overflow", "x-queue-type"} else "long"
):
raise DeliveryError("broker argument type differs")
result[name] = argument
else:
raise DeliveryError("broker argument shape differs")
if "x-queue-type" in result and result.pop("x-queue-type") != "classic":
raise DeliveryError("actual broker queue type differs")
if canonical(result) != canonical(dependencies.QUEUE_ARGUMENTS):
raise DeliveryError("actual broker queue budgets differ")
return result
@@ -148,49 +125,6 @@ def validate_snapshot(value: dict) -> dict:
}
def command(name: str):
if name not in QUERIES:
raise DeliveryError("unknown broker metadata query")
# A file-backed capped stream prevents malicious metadata/log output from
# allocating an unbounded capture buffer. The unit adds a hard file limit.
env = {
"PATH": "/usr/bin:/bin",
"LC_ALL": "C.UTF-8",
"HOME": dependencies.STORAGE["rabbitmq"]["path"],
"ERL_EPMD_ADDRESS": "127.0.0.1",
"RABBITMQ_NODENAME": "kf@localhost",
"RABBITMQ_CTL_ERL_ARGS": (
"+S 1:1 +A 2 +P 1024 +Q 1024 -kernel inet_dist_use_interface {127,0,0,1}"
),
}
with tempfile.TemporaryFile() as output:
try:
result = subprocess.run(
[
str(CTL),
"-q",
"-n",
"kf@localhost",
"--timeout",
"5",
*QUERIES[name],
"--formatter=json",
],
stdout=output,
stderr=subprocess.DEVNULL,
timeout=8,
env=env,
)
except subprocess.TimeoutExpired as error:
raise DeliveryError("broker " + name + " query exceeded deadline") from error
if result.returncode:
raise DeliveryError("broker " + name + " query failed")
if output.tell() > 32768:
raise DeliveryError("broker " + name + " query output exceeds bound")
output.seek(0)
return operator_units.decode(output.read(32769))
def observe() -> dict:
operator_units.guarded("broker-health")
preflight.protected(CTL)
@@ -202,8 +136,7 @@ def observe() -> dict:
or (info.st_uid, info.st_gid, stat.S_IMODE(info.st_mode)) != (1204, 1204, 0o400)
):
raise DeliveryError("ordinary private broker cookie required")
data = {name: command(name) for name in QUERIES}
return validate_snapshot(data)
return validate_snapshot(broker_snapshot.command())
def main() -> None:
scripts/release/broker_snapshot.py
+96
View File
@@ -0,0 +1,96 @@
"""One fixed read-only RabbitMQ 4.3.6 metadata RPC; no caller expressions."""
import subprocess
import tempfile
from pathlib import Path
from . import dependency_policy, operator_units
from .policy import DeliveryError, fields
CTL = Path("/usr/lib/rabbitmq/sbin/rabbitmqctl")
# Fixed getters only: no caller text, mutation, message reads or password hashes.
SNAPSHOT_EXPRESSION = r"""
rpc:call(node(), erlang, apply, [fun() ->
Count = rabbit_amqqueue:count(<<"kf">>),
true = (Count =:= 0 orelse Count =:= 1),
Normalize = fun
(name, {resource, <<"kf">>, queue, Value}) -> Value;
(arguments, Value) -> [[K, atom_to_binary(T, utf8), V] || {K,T,V} <- Value];
(type, rabbit_classic_queue) -> <<"classic">>;
(K, Value) when K =:= policy; K =:= operator_policy ->
case Value of undefined -> <<>>; none -> <<>>; '' -> <<>>; _ -> Value end;
(effective_policy_definition, Value) when is_list(Value) -> maps:from_list(Value);
(tags, Value) -> [case T of
A when is_atom(A) -> atom_to_binary(A,utf8); _ -> T end || T <- Value];
(_, Value) when is_atom(Value), Value =/= true, Value =/= false ->
atom_to_binary(Value,utf8);
(_, Value) -> Value
end,
ToMap = fun(Items) -> maps:from_list(
[{atom_to_binary(K,utf8),Normalize(K,V)} || {K,V} <- Items]) end,
Queues = [ToMap(Row) || Row <- rabbit_amqqueue:info_all(<<"kf">>,
[name,durable,auto_delete,exclusive,type,state,arguments,messages_ready,
messages_unacknowledged,message_bytes,consumers,policy,operator_policy,
effective_policy_definition])],
Limits = [#{<<"vhost">> => V,
<<"limits">> => iolist_to_binary(rabbit_json:encode(maps:from_list(L)))}
|| {V,L} <- rabbit_vhost_limit:list()],
Users = [ToMap(Row) || Row <- rabbit_auth_backend_internal:list_users()],
Permissions = [ToMap(Row) || Row <-
rabbit_auth_backend_internal:list_vhost_permissions(<<"kf">>)],
iolist_to_binary(rabbit_json:encode(#{<<"queues">> => Queues, <<"vhost_limits">> => Limits,
<<"users">> => Users, <<"permissions">> => Permissions}))
end, []], 5000).
""".strip()
def command() -> dict:
env = {
"PATH": "/usr/bin:/bin",
"LC_ALL": "C.UTF-8",
"HOME": dependency_policy.STORAGE["rabbitmq"]["path"],
"ERL_EPMD_ADDRESS": "127.0.0.1",
"RABBITMQ_NODENAME": "kf@localhost",
"RABBITMQ_CTL_ERL_ARGS": (
"+S 1:1 +A 2 +P 1024 +Q 1024 -kernel inet_dist_use_interface {127,0,0,1}"
),
}
with tempfile.TemporaryFile() as output:
try:
result = subprocess.run(
[
str(CTL),
"-q",
"-n",
"kf@localhost",
"--timeout",
"5",
"eval",
SNAPSHOT_EXPRESSION,
"--formatter=json",
],
stdout=output,
stderr=subprocess.DEVNULL,
timeout=30,
env=env,
)
except subprocess.TimeoutExpired as error:
raise DeliveryError("broker snapshot query exceeded deadline") from error
if result.returncode:
raise DeliveryError("broker snapshot query failed")
if output.tell() > 32768:
raise DeliveryError("broker snapshot query output exceeds bound")
output.seek(0)
value = operator_units.decode(output.read(32769))
fields(value, {"result", "node", "value"}, "broker snapshot envelope")
if (
value["result"] != "ok"
or value["node"] != "kf@localhost"
or not isinstance(value["value"], str)
):
raise DeliveryError("broker snapshot result or node differs")
return fields(
operator_units.decode(value["value"].encode()),
{"queues", "vhost_limits", "users", "permissions"},
"broker snapshot",
)
scripts/release/node_health.py
+1
View File
@@ -66,6 +66,7 @@ def binding() -> tuple[dict, dict, Path]:
"availability.py",
"node_health.py",
"broker_health.py",
"broker_snapshot.py",
"factory_health.py",
"import_health.py",
"operator_units.py",
scripts/release/operator_units.py
+4 -4
View File
@@ -11,10 +11,10 @@ from .policy import DeliveryError, invalid_constant, unique_object
PROFILES = {
"node-health": ("root", 0, 128, 16, 50, 300),
"broker-health": ("rabbitmq", 1204, 256, 64, 50, 45),
"factory-health": ("kf-worker", 1202, 32, 4, 10, 30),
"import-health": ("kf-oj", 1200, 128, 16, 25, 60),
"outbox-health": ("kf-oj", 1200, 128, 16, 25, 60),
"broker-health": ("rabbitmq", 1204, 256, 64, 100, 45),
"factory-health": ("kf-worker", 1202, 32, 4, 100, 30),
"import-health": ("kf-oj", 1200, 128, 16, 100, 60),
"outbox-health": ("kf-oj", 1200, 128, 16, 100, 60),
"node-maintenance": ("root", 0, 256, 32, 50, 900),
"node-observation": ("root", 0, 128, 16, 25, 360),
}