Kronecker/KF-OnlineJudge
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(delivery): preserve bounded operator failure diagnostics

Browse Source
This commit is contained in:
Kronecker
2026-10-30 02:01:16 +08:00
parent cde9fab2d2
commit c820d4b637
6 changed files with 429 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/release/README.md
+9
View File
@@ -341,6 +341,15 @@ the outer operator also closes admission. If that record cannot be written, it
stops only the verified API unit. A refusal before dispatch, such as an already
active helper, does not change admission or stop another operation.
Dispatched failures append one `KF_OPERATOR_DIAGNOSTIC` JSON line to the CLI
error. Its versioned, bounded chain records fixed helper names, failure stages,
exit codes, output sizes and SHA-256 digests. Failed readiness includes only
allowlisted component error codes and at most two dependency helper records.
Raw child output, arguments, credentials and exception text are excluded from
the diagnostic line. Malformed, oversized or duplicate child diagnostic lines
are discarded. This observation does not change service limits, deadlines,
admission checks or cleanup requirements; a failed start remains a failure.
Drain keeps API available to accepted work and polls Coordinator, outbox, import,
broker and factory within a finite deadline. Untruncated owner counts, stopped
process groups, empty tasks/identity/capture resources and an absent factory socket
scripts/release/lifecycle.py
+8 -3
View File
@@ -639,12 +639,16 @@ class Maintenance:
self.start_group(group)
ready = operator_units.run("node-health", self.payload)
if ready.get("service_preconditions_met") is not True:
raise DeliveryError("service preconditions unavailable; admission remains closed")
raise operator_units.readiness_failure(
"service preconditions unavailable; admission remains closed", ready
)
try:
availability.write(self.selected, "open")
ready = operator_units.run("node-health", self.payload)
if ready.get("ready_for_admission") is not True:
raise DeliveryError("admission readiness readback unconfirmed")
raise operator_units.readiness_failure(
"admission readiness readback unconfirmed", ready, admission=True
)
self.save("accepting")
except (OSError, ValueError, subprocess.SubprocessError):
self.close_failed_admission()
@@ -751,7 +755,8 @@ def main() -> None:
1,
"Lifecycle refused: "
+ (str(error) if isinstance(error, DeliveryError) else type(error).__name__)
+ "\n",
+ "\n"
+ operator_units.diagnostic_suffix(error),
)
scripts/release/node_health.py
+17 -5
View File
@@ -525,6 +525,13 @@ def observe(*, probe: bool) -> dict:
"errors": [],
}
components, errors = result["components"], result["errors"]
helper_failures = []
def remember_helper(error):
if isinstance(error, operator_units.OperatorExecutionError):
helper_failures.extend(error.failures[: max(0, 2 - len(helper_failures))])
result["operator_failures"] = helper_failures
try:
components["journal"] = journal.check_ready()
except (OSError, ValueError, subprocess.SubprocessError):
@@ -580,13 +587,15 @@ def observe(*, probe: bool) -> dict:
with _status_helpers(payload) as pending:
try:
components["broker"] = pending["broker-health"].result()
except (OSError, ValueError, subprocess.SubprocessError):
except (OSError, ValueError, subprocess.SubprocessError) as error:
remember_helper(error)
errors.append("broker_metadata_unavailable")
try:
state = pending["outbox-health"].result()
validate_outbox(state)
components["outbox_pipeline"] = state
except (OSError, ValueError, subprocess.SubprocessError):
except (OSError, ValueError, subprocess.SubprocessError) as error:
remember_helper(error)
errors.append("outbox_pipeline_unavailable")
try:
state = pending["import-health"].result()
@@ -617,13 +626,15 @@ def observe(*, probe: bool) -> dict:
if type(state[key]) is not int or not 0 <= state[key] <= maximum:
raise DeliveryError("import status bound differs")
components["import_pipeline"] = state
except (OSError, ValueError, subprocess.SubprocessError):
except (OSError, ValueError, subprocess.SubprocessError) as error:
remember_helper(error)
errors.append("import_pipeline_unavailable")
try:
status = pending["factory-health"].result()
validate_factory(status, digest(paths["runtime.json"]))
components["factory_status"] = status
except (OSError, ValueError, subprocess.SubprocessError):
except (OSError, ValueError, subprocess.SubprocessError) as error:
remember_helper(error)
errors.append("factory_status_unavailable")
if probe:
try:
@@ -723,7 +734,8 @@ def main() -> None:
1,
"Node health refused: "
+ (str(error) if isinstance(error, DeliveryError) else type(error).__name__)
+ "\n",
+ "\n"
+ operator_units.diagnostic_suffix(error),
)
scripts/release/operator_units.py
+167 -1
View File
@@ -1,7 +1,9 @@
"""Finite fixed-purpose operator helpers; no caller-controlled executable or identity."""
import hashlib
import json
import os
import re
import resource
import subprocess
from pathlib import Path
@@ -13,6 +15,10 @@ from .policy import DeliveryError, invalid_constant, unique_object
class OperatorExecutionError(DeliveryError):
"""A fixed helper was dispatched but its result or cleanup was unconfirmed."""
def __init__(self, message: str, failures: list | None = None):
super().__init__(message)
self.failures = failures or []
PROFILES = {
"node-health": ("root", 0, 128, 16, 50, 300),
@@ -28,6 +34,156 @@ PROFILES = {
# OTP 27 reserves a 32 MiB dual-mapped JIT block before serving CLI queries.
FILE_LIMIT_MIB = {"broker-health": 64}
DIAGNOSTIC_PREFIX = "KF_OPERATOR_DIAGNOSTIC "
READINESS_ERRORS = frozenset(
{
"bounded_journal_unavailable",
"https_entry_unavailable",
"selected_binding_changed",
"db_authority_unavailable",
"api_live_unavailable",
"execution_owner_unavailable",
"broker_metadata_unavailable",
"outbox_pipeline_unavailable",
"import_pipeline_unavailable",
"factory_status_unavailable",
"notification_delivery_unconfirmed",
*(
name + "_unavailable"
for name in (
"api",
"coordinator",
"outbox",
"problem-import",
"worker",
"postgres",
"rabbitmq",
"sandbox-factory",
"proxy",
)
),
}
)
def failure_record(name: str, outcome: str, result=None, error=None) -> dict:
"""Record only fixed identifiers, sizes and digests; never child output or argv."""
record = {"helper": name, "outcome": outcome, "returncode": None}
if result is not None:
record["returncode"] = result.returncode
for stream in ("stdout", "stderr"):
value = getattr(result, stream, b"") if result is not None else b""
if isinstance(error, subprocess.TimeoutExpired):
value = getattr(error, stream, None) or b""
record[stream + "_bytes"] = len(value)
record[stream + "_sha256"] = hashlib.sha256(value).hexdigest()
record["error_sha256"] = hashlib.sha256(
(str(error) if isinstance(error, DeliveryError) else type(error).__name__).encode()
).hexdigest()
return record
def diagnostic_chain(stderr: bytes) -> list:
"""Accept one small, strictly shaped record from a fixed child CLI."""
if len(stderr) > 4096:
return []
lines = [line for line in stderr.splitlines() if line.startswith(DIAGNOSTIC_PREFIX.encode())]
try:
if len(lines) != 1:
return []
value = decode(lines[0][len(DIAGNOSTIC_PREFIX) :], 3072)
if (
set(value) != {"schema_version", "failures"}
or type(value["schema_version"]) is not int
or value["schema_version"] != 1
):
return []
failures = value["failures"]
if not isinstance(failures, list) or not 1 <= len(failures) <= 3:
return []
fields = {
"helper",
"outcome",
"returncode",
"stdout_bytes",
"stdout_sha256",
"stderr_bytes",
"stderr_sha256",
"error_sha256",
}
for row in failures:
if not isinstance(row, dict) or set(row) not in (
fields,
fields | {"readiness_errors"},
):
return []
if row["helper"] not in PROFILES or row["outcome"] not in {
"dispatch",
"exit",
"decode",
"cleanup",
"timeout",
"preconditions",
"admission",
}:
return []
code = row["returncode"]
if code is not None and (type(code) is not int or not -255 <= code <= 255):
return []
if any(
type(row[k]) is not int or not 0 <= row[k] <= 2**63 - 1
for k in ("stdout_bytes", "stderr_bytes")
):
return []
if any(
not isinstance(row[k], str) or not re.fullmatch("[a-f0-9]{64}", row[k])
for k in ("stdout_sha256", "stderr_sha256", "error_sha256")
):
return []
errors = row.get("readiness_errors", [])
if (
not isinstance(errors, list)
or len(errors) > len(READINESS_ERRORS)
or any(not isinstance(code, str) or code not in READINESS_ERRORS for code in errors)
):
return []
return failures
except (TypeError, ValueError, KeyError):
return []
def diagnostic_suffix(error: Exception) -> str:
if not isinstance(error, OperatorExecutionError) or not error.failures:
return ""
# Revalidate even internally built records before they cross a CLI boundary.
value = json.dumps({"schema_version": 1, "failures": error.failures}, separators=(",", ":"))
line = DIAGNOSTIC_PREFIX + value
if len(error.failures) > 4 or len(line) > 3072:
return ""
# The child decoder allows three ancestors; validate the fourth separately.
for row in error.failures:
single = DIAGNOSTIC_PREFIX + json.dumps({"schema_version": 1, "failures": [row]})
if not diagnostic_chain(single.encode()):
return ""
return line + "\n"
def readiness_failure(message: str, result: dict, *, admission: bool = False):
error = OperatorExecutionError(message)
row = failure_record("node-health", "admission" if admission else "preconditions", error=error)
errors = result.get("errors")
if (
isinstance(errors, list)
and len(errors) <= len(READINESS_ERRORS)
and all(isinstance(code, str) and code in READINESS_ERRORS for code in errors)
):
row["readiness_errors"] = errors
child = DIAGNOSTIC_PREFIX + json.dumps(
{"schema_version": 1, "failures": result.get("operator_failures", [])}
)
error.failures = diagnostic_chain(child.encode())[:2] + [row]
return error
def guarded(name: str) -> None:
_, uid, memory, pids, cpu, _ = PROFILES[name]
@@ -239,6 +395,8 @@ def run(
command.append("--probe-notification")
elif notification_probe:
raise DeliveryError("unexpected broker probe override")
result = None
outcome = "dispatch"
try:
try:
# Pipe output is bounded by the fixed helper and the service file/memory
@@ -249,12 +407,15 @@ def run(
timeout=seconds + 15,
env={"PATH": "/usr/bin:/bin", "LC_ALL": "C"},
)
outcome = "exit"
if result.returncode or len(result.stderr) > 4096:
raise DeliveryError("operator helper unavailable")
outcome = "decode"
value = decode(result.stdout)
if not isinstance(value, dict):
raise DeliveryError("operator helper object required")
finally:
previous_outcome, outcome = outcome, "cleanup"
actual = dict(
line.split("=", 1)
for line in service_install.systemctl(
@@ -277,7 +438,12 @@ def run(
)
if actual.get("ActiveState") != "inactive" or actual.get("MainPID") != "0":
raise DeliveryError("operator helper cleanup unconfirmed")
outcome = previous_outcome
except (OSError, ValueError, subprocess.SubprocessError) as error:
message = str(error) if isinstance(error, DeliveryError) else type(error).__name__
raise OperatorExecutionError(message) from error
if isinstance(error, subprocess.TimeoutExpired) and outcome != "cleanup":
outcome = "timeout"
failures = diagnostic_chain(result.stderr) if result is not None else []
failures.append(failure_record(name, outcome, result, error))
raise OperatorExecutionError(message, failures) from error
return value