feat(acceptance): 添加失败关闭的第三阶段验收运行器

This commit is contained in:
2026-09-09 08:53:53 +08:00
parent b0783c9356
commit 31ddb8d49c
8 changed files with 691 additions and 1 deletions
+1
View File
@@ -30,6 +30,7 @@ jobs:
working-directory: backend
- run: uv run pytest
working-directory: backend
- run: python scripts/phase3-production-acceptance.py --list-cases --json
service-test:
runs-on: ubuntu-latest
@@ -0,0 +1,133 @@
from __future__ import annotations
import importlib.util
import json
from argparse import Namespace
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
SPEC = importlib.util.spec_from_file_location("phase3_acceptance", ROOT / "scripts" / "phase3_acceptance.py")
assert SPEC and SPEC.loader
runner = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(runner)
def config(path: Path, data_root: Path, **changes) -> Path:
payload = {
"schema": 1,
"run_id": "runner-test-001",
"isolated": True,
"allow_destructive": True,
"platform_profile": "windows-11-x64",
"data_root": str(data_root),
"seed": 20260908,
"service_urls": {},
"artifacts": {},
"secret_env": {},
}
payload.update(changes)
path.write_text(json.dumps(payload), encoding="utf-8")
return path
def test_manifest_contains_every_documented_case_once():
assert len(runner.ALL_CASES) == 30
assert len(set(runner.ALL_CASES)) == 30
assert runner.select_cases("sync-client", "s-08") == ("S-08",)
with pytest.raises(runner.AcceptanceError, match="CASE_NOT_IN_SUITE"):
runner.select_cases("sidecar", "S-08")
def test_config_rejects_personal_vault_plaintext_secrets_and_unconfirmed_roots(tmp_path):
with pytest.raises(runner.AcceptanceError, match="PERSONAL_VAULT"):
runner.load_config(config(tmp_path / "vault.json", runner.VAULT_ROOT))
with pytest.raises(runner.AcceptanceError, match="PLAINTEXT_SECRET"):
runner.load_config(config(tmp_path / "secret.json", tmp_path / "data", password="do-not-store-this"))
existing = tmp_path / "existing"
existing.mkdir()
(existing / "keep.txt").write_text("keep", encoding="utf-8")
loaded = runner.load_config(config(tmp_path / "existing.json", existing))
with pytest.raises(runner.AcceptanceError, match="NOT_EMPTY_OR_MARKED"):
runner.prepare_isolated_root(loaded)
assert (existing / "keep.txt").read_text(encoding="utf-8") == "keep"
def test_missing_driver_is_a_junit_failure_and_never_a_skip(tmp_path):
data_root = tmp_path / "isolated"
config_path = config(tmp_path / "config.json", data_root)
report = tmp_path / "report"
args = Namespace(
suite="sidecar", case="A-01", config=str(config_path), report_dir=str(report),
list_cases=False, json=False,
)
assert runner.execute(args, {}) == 1
result = json.loads((report / "cases" / "A-01.json").read_text(encoding="utf-8"))
summary = json.loads((report / "summary.json").read_text(encoding="utf-8"))
junit = (report / "junit.xml").read_text(encoding="utf-8")
assert result["status"] == "NOT_IMPLEMENTED"
assert summary["status"] == "NOT_PASSED"
assert summary["contains_credentials"] is False
assert '<failure type="NOT_IMPLEMENTED">' in junit
assert "skipped=\"0\"" in junit
def test_driver_result_must_supply_assertions_metrics_and_zero_exit(tmp_path):
driver = tmp_path / "driver.py"
driver.write_text("", encoding="utf-8")
original_root = runner.ROOT
try:
runner.ROOT = tmp_path
case = {
"driver": "driver.py",
"required_metrics": ("peak_rss_bytes",),
}
result_root = tmp_path / "result"
result_root.mkdir()
result = runner.run_case("A-01", tmp_path / "config.json", {"data_root": str(tmp_path / "data"), "secret_env": {}}, result_root, {}, {"A-01": case})
assert result["status"] == "FAILED"
assert "RESULT_ASSERTIONS_MISSING" in result["runner_errors"]
assert "RESULT_METRIC_MISSING:peak_rss_bytes" in result["runner_errors"]
finally:
runner.ROOT = original_root
def test_valid_driver_passes_and_its_log_is_redacted(tmp_path):
driver = tmp_path / "driver.py"
driver.write_text(
"""import argparse, json, os
from pathlib import Path
p=argparse.ArgumentParser();p.add_argument('--config');p.add_argument('--output');a=p.parse_args()
print(os.environ['TEST_ACCEPTANCE_SECRET'])
Path(a.output).write_text(json.dumps({
'schema':1,'case_id':os.environ['OPENNEXUS_ACCEPTANCE_CASE_ID'],'status':'PASSED','reason':'',
'assertions':[{'name':'independent oracle','status':'PASSED','evidence':'fixture'}],
'metrics':{'peak_rss_bytes':123,'max_process_count':1,'denied_access_count':0},
'files':[],'revisions':[]}), encoding='utf-8')
""",
encoding="utf-8",
)
original_root = runner.ROOT
try:
runner.ROOT = tmp_path
result_root = tmp_path / "result"
result_root.mkdir()
result = runner.run_case(
"A-01",
tmp_path / "config.json",
{
"data_root": str(tmp_path / "data"), "secret_env": {"fixture": "TEST_ACCEPTANCE_SECRET"},
"platform_profile": "windows-11-x64", "artifacts": {},
},
result_root,
{"TEST_ACCEPTANCE_SECRET": "planted-secret-value"},
{"A-01": {"driver": "driver.py", "required_metrics": ("peak_rss_bytes",)}},
)
assert result["status"] == "PASSED"
log = (result_root / "logs" / "A-01.log").read_text(encoding="utf-8")
assert "planted-secret-value" not in log
assert "[REDACTED]" in log
finally:
runner.ROOT = original_root
@@ -182,7 +182,7 @@ cargo test --manifest-path frontend/src-tauri/Cargo.toml --lib --locked
cargo clippy --manifest-path frontend/src-tauri/Cargo.toml --lib --locked -- -D warnings
```
以下验收runner和配置是**P0/R0待交付接口,现在不存在,不能报告已运行**:仓库根执行`python scripts/phase3-production-acceptance.py --suite <suite> --config <isolated-config> --report-dir <output>`suite取`sidecar/credentials/sandbox/extensions/sync-client/sync-service/e2e/all`,支持`--case <ID>`单例复跑。配置由环境模板产生,含临时数据根、服务URL、安装包路径、平台profile、seed及秘密的环境引用,不能含生产凭据。runner必须拒绝仓库个人Vault未标记隔离环境,缺依赖/安装包/权限时非零退出;必测项skip也非零,不能只打印成功。
验收入口已经建立:仓库根执行`python scripts/phase3-production-acceptance.py --suite <suite> --config <isolated-config> --report-dir <output>`suite取`sidecar/credentials/sandbox/extensions/sync-client/sync-service/e2e/all`,支持`--case <ID>`单例复跑。配置由[隔离模板](../../scripts/phase3-acceptance-config.example.json)产生,含临时数据根、服务URL、安装包路径、平台profile、seed及秘密的环境引用,不能含生产凭据。runner拒绝仓库个人Vault未标记隔离环境和非空报告目录;缺case driver、依赖/安装包/权限必测项skip均以非零退出,不能只打印成功。当前各验收 ID 的完整生产 driver 仍须逐项接入,未登记项明确报告 `NOT_IMPLEMENTED`,不因 runner 接口存在而视为已验收;使用和结果契约见[生产验收 Runner](../development/OpenNexus生产验收Runner.md)。
runner输出JUnit、逐ID JSON、耗时/峰值内存、进程树/拒绝访问计数、文件摘要与revision清单及脱敏日志。退出0仅表示该suite所有适用ID通过;`all`要求显式平台profile并检查缺失项。下表同一行内所有断言均必测,每类输入保留独立子用例;测试代码硬编码预期协议向量或独立oracle,不能用被测函数自证。P0需生成ID清单并校验报告完整性。
@@ -772,3 +772,12 @@ Core 的独立数据目录目前不等于已授权 Vault。Python 旧笔记写
- 实际 Python Core + Rust Host 集成覆盖创建、20 次相同重放、改变输入冲突、列表、跨 Vault 拒绝、CAS 更新、过期摘要拒绝、Agent 选择运行、删除及关闭前 journal 计数。实际 Uvicorn Sync + 两个 Rust Workspace 客户端对用户 Skill 进行了 60 轮同时修改,保留本地/采用远端/创建文本副本各 20 轮,并验证旧摘要拒绝、重复解决、两端收敛、重开稳定;第三个默认范围客户端可接收用户 Skill,同时仍排除未选择的人设与布局。
- 最终 desktop Rust 全目标 148 通过、12 ignoredlibrary 131、Host 10、其余集成 7),Clippy `--all-targets -D warnings` 通过,日志 `.build/user-skill-rust-full-final.log``.build/user-skill-clippy-current.log`。后端 908 项通过、1 项依赖弃用警告,日志 `.build/user-skill-python-full-current.log`。前端 103 个测试文件、536 项通过,类型检查和生产构建通过,日志 `.build/user-skill-frontend-full-final2.log``.build/user-skill-frontend-build-final2.log`
- 证据来自本机隔离服务和客户端进程,尚未替代两台独立硬件上的桌面 UI、跨发布版本或每个持久化边界强制终止验收。Provider 非秘密参数、安装清单、对话等规划内逻辑数据仍未适配;完整生产化目标保持未完成。
## 增量:生产验收 Runner 的失败闭合入口
- 新增规划约定的 `scripts/phase3-production-acceptance.py`,固定登记 sidecar/credentials/sandbox/extensions/sync-client/sync-service/e2e 七个 suite 和 30 个验收 ID,支持 `--case` 单例与机器可读清单。CI 的 backend job 会验证清单入口,新单元测试覆盖清单唯一性和 suite 归属。
- 配置必须显式确认隔离和破坏性测试,提供唯一 run、平台 profile、固定 seed 及绝对临时数据根;首次只接受空目录并写入绑定标记。runner 拒绝个人 Vault、仓库或 Git 路径、符号链接、非空未标记目录、明文秘密字段、带认证信息/query/fragment 的服务 URL,以及未显式声明的测试 HTTP。秘密只按环境变量名引用,缺值直接失败。
- 每个 case 只调用仓库固定登记的 driver,配置不能注入命令。driver 的 JSON 必须含独立断言、文件/revision 清单和该 ID 要求的指标;非零退出、超时、坏 schema、缺指标、缺安装产物/权限/秘密引用、必测 skip 或结果秘密命中均失败。输出含逐 ID JSON、JUnit、固定 ID 清单、脱敏限长日志、commit、锁文件/安装产物/config 摘要;已有非空报告目录不覆盖。
- 5 项 runner 测试通过,覆盖个人 Vault/明文秘密/非空目录拒绝,未登记 ID 生成 JUnit failure 且退出 1driver 缺断言/指标失败,以及测试秘密在日志中被替换。实际 CLI 清单检查通过;使用模板运行 A-01 得到预期 `NOT_IMPLEMENTED`、summary `NOT_PASSED` 和退出 1,证明缺实现不会误报成功。包含该测试的后端全套 913 项通过、1 项依赖弃用警告,日志 `.build/acceptance-runner-python.log`Python 编译与 Markdown 链接检查通过。
- 当前仅完成 P0 runner 与报告协议,30 项完整生产 driver 尚未登记;因此没有任何 A/B/C/D/S/E ID 因本增量变为通过。后续逐项接入时仍须满足规划表的全部断言、实机/生产依赖和量化指标,完整生产化目标保持未完成。
@@ -0,0 +1,27 @@
# OpenNexus 生产验收 Runner
`scripts/phase3-production-acceptance.py` 是第三阶段 A/B/C/D/S/E 验收 ID 的统一入口。它只把仓库登记的逐 ID driver 结果计为生产验收;普通单元测试、隔离冒烟和历史人工汇总不会自动换算成通过。
查看固定的 30 项清单:
```powershell
python scripts/phase3-production-acceptance.py --list-cases --json
```
复制 `scripts/phase3-acceptance-config.example.json` 到仓库外,设置唯一 `run_id` 和平台 profile。`data_root` 可直接使用绝对路径,也可像模板一样通过环境变量引用。第一次运行只接受不存在或空目录并写入与 `run_id` 绑定的隔离标记;后续只能由同一 run 复用。个人 Vault、仓库或 `.git` 的父目录/子目录、符号链接、含明文密码/令牌的配置均会被拒绝。HTTP 服务只允许在配置中明确标记为测试服务,URL 不得包含用户信息、query 或 fragment。
执行单例时应使用新的空报告目录:
```powershell
$env:OPENNEXUS_ACCEPTANCE_DATA_ROOT = 'D:\opennexus-acceptance\run-001'
python scripts/phase3-production-acceptance.py `
--suite sync-service --case S-04 `
--config D:\opennexus-acceptance\run-001.json `
--report-dir D:\opennexus-acceptance\reports\run-001-s04
```
每个 driver 由仓库内 `CASE_DRIVERS` 固定登记,接收 `--config``--output`,并通过环境获得 case ID、数据根和报告根。结果必须是 schema 1 JSON,包含匹配的 `case_id``PASSED/FAILED/SKIPPED/NOT_APPLICABLE` 状态、至少一项独立断言、该 ID 要求的非负整数指标,以及 `files``revisions` 清单。只有 driver 退出 0、结果为 `PASSED`、所有必填指标齐全且结果不含配置引用的秘密值时,该 ID 才通过。缺 driver、必测 skip、非零退出、超时、缺安装包/权限/秘密环境引用、坏 schema 或秘密命中都会让 suite 非零退出。
报告目录包含 `summary.json``case-manifest.json``junit.xml``cases/<ID>.json` 和脱敏的 `logs/<ID>.log`。摘要记录 commit、各锁文件 SHA-256、配置摘要与已提供安装产物摘要。日志将仓库、数据根、报告根、用户主目录和配置声明的秘密值替换为占位符,并限制为 10 MiB。报告目录必须为空,避免单例复跑覆盖原始证据。
当前 runner 与失败闭合行为已实现,各生产验收 ID 的完整 driver 尚未登记。因此现在运行任一 ID 会生成 `NOT_IMPLEMENTED` 证据并退出 1;这用于阻止误报,不是验收通过。
@@ -0,0 +1,15 @@
{
"schema": 1,
"run_id": "replace-with-unique-run-id",
"isolated": true,
"allow_destructive": true,
"platform_profile": "windows-11-x64",
"data_root_env": "OPENNEXUS_ACCEPTANCE_DATA_ROOT",
"seed": 20260908,
"allow_http_test_services": true,
"service_urls": {
"sync": "http://127.0.0.1:18080"
},
"artifacts": {},
"secret_env": {}
}
+7
View File
@@ -0,0 +1,7 @@
"""Documented command entry point for the production acceptance runner."""
from phase3_acceptance import main
if __name__ == "__main__":
raise SystemExit(main())
+498
View File
@@ -0,0 +1,498 @@
"""Fail-closed runner for the OpenNexus phase-three production acceptance IDs."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
from xml.etree import ElementTree
ROOT = Path(__file__).resolve().parents[1]
VAULT_ROOT = (ROOT / "backend" / "data" / "vault").resolve()
SCHEMA = 1
CASE_SUITES = {
"sidecar": tuple(f"A-{number:02d}" for number in range(1, 5)),
"credentials": tuple(f"B-{number:02d}" for number in range(1, 5)),
"sandbox": tuple(f"C-{number:02d}" for number in range(1, 5)),
"extensions": tuple(f"D-{number:02d}" for number in range(1, 5)),
"sync-client": ("S-01", "S-02", "S-03", "S-08"),
"sync-service": ("S-04", "S-05", "S-06", "S-07", "S-09"),
"e2e": tuple(f"E-{number:02d}" for number in range(1, 6)),
}
ALL_CASES = tuple(case for cases in CASE_SUITES.values() for case in cases)
# A case becomes executable only when a repository-owned driver is registered here.
# Component/unit test commands are deliberately not treated as production acceptance.
CASE_DRIVERS: dict[str, dict[str, Any]] = {}
ENV_NAME = re.compile(r"[A-Z][A-Z0-9_]{2,127}")
RUN_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{2,63}")
SENSITIVE_KEY = re.compile(r"(?:password|passwd|secret|token|api[_-]?key|credential)", re.I)
MAX_LOG_BYTES = 10 * 1024 * 1024
class AcceptanceError(ValueError):
"""A stable, user-actionable runner configuration error."""
def _utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def _inside(path: Path, parent: Path) -> bool:
try:
path.relative_to(parent)
return True
except ValueError:
return False
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _contains_plain_secret(value: Any, path: tuple[str, ...] = ()) -> bool:
if isinstance(value, dict):
for key, item in value.items():
nested = path + (str(key),)
if SENSITIVE_KEY.search(str(key)) and str(key) != "secret_env" and (not path or path[-1] != "secret_env"):
return True
if _contains_plain_secret(item, nested):
return True
elif isinstance(value, list):
return any(_contains_plain_secret(item, path) for item in value)
return False
def _resolve_data_root(config: dict[str, Any], environ: dict[str, str]) -> Path:
direct = config.get("data_root")
reference = config.get("data_root_env")
if bool(direct) == bool(reference):
raise AcceptanceError("CONFIG_DATA_ROOT_REQUIRED: set exactly one of data_root or data_root_env")
if reference:
if not isinstance(reference, str) or not ENV_NAME.fullmatch(reference):
raise AcceptanceError("CONFIG_DATA_ROOT_ENV_INVALID")
direct = environ.get(reference)
if not direct:
raise AcceptanceError(f"CONFIG_ENV_MISSING: {reference}")
path = Path(str(direct)).expanduser()
if not path.is_absolute():
raise AcceptanceError("CONFIG_DATA_ROOT_NOT_ABSOLUTE")
resolved = path.resolve(strict=False)
if resolved == ROOT or _inside(ROOT, resolved):
raise AcceptanceError("CONFIG_DATA_ROOT_CONTAINS_REPOSITORY")
if _inside(resolved, VAULT_ROOT) or _inside(VAULT_ROOT, resolved):
raise AcceptanceError("CONFIG_PERSONAL_VAULT_FORBIDDEN")
if _inside(resolved, (ROOT / ".git").resolve()):
raise AcceptanceError("CONFIG_GIT_DIRECTORY_FORBIDDEN")
if path.exists() and path.is_symlink():
raise AcceptanceError("CONFIG_DATA_ROOT_SYMLINK_FORBIDDEN")
return resolved
def load_config(path: Path, environ: dict[str, str] | None = None) -> dict[str, Any]:
environ = os.environ if environ is None else environ
try:
config = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as error:
raise AcceptanceError("CONFIG_UNREADABLE") from error
if not isinstance(config, dict) or config.get("schema") != SCHEMA:
raise AcceptanceError("CONFIG_SCHEMA_UNSUPPORTED")
if _contains_plain_secret(config):
raise AcceptanceError("CONFIG_PLAINTEXT_SECRET_FORBIDDEN")
allowed = {
"schema", "run_id", "isolated", "allow_destructive", "platform_profile",
"data_root", "data_root_env", "seed", "allow_http_test_services",
"service_urls", "artifacts", "secret_env",
}
unknown = sorted(set(config) - allowed)
if unknown:
raise AcceptanceError(f"CONFIG_FIELD_UNKNOWN: {','.join(unknown)}")
if config.get("isolated") is not True or config.get("allow_destructive") is not True:
raise AcceptanceError("CONFIG_ISOLATION_CONFIRMATION_REQUIRED")
run_id = config.get("run_id")
if not isinstance(run_id, str) or not RUN_ID.fullmatch(run_id):
raise AcceptanceError("CONFIG_RUN_ID_INVALID")
profile = config.get("platform_profile")
if not isinstance(profile, str) or not profile.strip():
raise AcceptanceError("CONFIG_PLATFORM_PROFILE_REQUIRED")
seed = config.get("seed")
if not isinstance(seed, int) or isinstance(seed, bool) or not 0 <= seed <= 2**63 - 1:
raise AcceptanceError("CONFIG_SEED_INVALID")
secret_env = config.get("secret_env", {})
if not isinstance(secret_env, dict):
raise AcceptanceError("CONFIG_SECRET_ENV_INVALID")
for alias, reference in secret_env.items():
if not isinstance(alias, str) or not isinstance(reference, str) or not ENV_NAME.fullmatch(reference):
raise AcceptanceError("CONFIG_SECRET_ENV_INVALID")
if not environ.get(reference):
raise AcceptanceError(f"CONFIG_ENV_MISSING: {reference}")
urls = config.get("service_urls", {})
if not isinstance(urls, dict):
raise AcceptanceError("CONFIG_SERVICE_URLS_INVALID")
for name, value in urls.items():
if not isinstance(name, str) or not isinstance(value, str):
raise AcceptanceError("CONFIG_SERVICE_URL_INVALID")
parsed = urlsplit(value)
if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:
raise AcceptanceError("CONFIG_SERVICE_URL_INVALID")
if parsed.query or parsed.fragment:
raise AcceptanceError("CONFIG_SERVICE_URL_INVALID")
if parsed.scheme == "http" and config.get("allow_http_test_services") is not True:
raise AcceptanceError("CONFIG_HTTP_SERVICE_REQUIRES_TEST_FLAG")
artifacts = config.get("artifacts", {})
if not isinstance(artifacts, dict) or any(not isinstance(name, str) or not isinstance(value, str) for name, value in artifacts.items()):
raise AcceptanceError("CONFIG_ARTIFACTS_INVALID")
normalized = dict(config)
normalized["data_root"] = str(_resolve_data_root(config, environ))
normalized.pop("data_root_env", None)
return normalized
def prepare_isolated_root(config: dict[str, Any]) -> Path:
root = Path(config["data_root"])
marker = root / ".opennexus-acceptance-isolated.json"
expected = {"schema": SCHEMA, "run_id": config["run_id"]}
if root.exists():
if root.is_symlink():
raise AcceptanceError("CONFIG_DATA_ROOT_SYMLINK_FORBIDDEN")
if not root.is_dir():
raise AcceptanceError("CONFIG_DATA_ROOT_NOT_DIRECTORY")
children = list(root.iterdir())
if children:
if not marker.is_file():
raise AcceptanceError("CONFIG_DATA_ROOT_NOT_EMPTY_OR_MARKED")
try:
actual = json.loads(marker.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as error:
raise AcceptanceError("CONFIG_ISOLATION_MARKER_INVALID") from error
if actual != expected:
raise AcceptanceError("CONFIG_ISOLATION_MARKER_MISMATCH")
else:
root.mkdir(parents=True)
marker.write_text(json.dumps(expected, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return root
def select_cases(suite: str, case: str | None) -> tuple[str, ...]:
selected = ALL_CASES if suite == "all" else CASE_SUITES[suite]
if case is None:
return selected
normalized = case.upper()
if normalized not in ALL_CASES:
raise AcceptanceError("CASE_UNKNOWN")
if normalized not in selected:
raise AcceptanceError("CASE_NOT_IN_SUITE")
return (normalized,)
def _redactor(config: dict[str, Any], report_root: Path, environ: dict[str, str]):
replacements = [
(str(ROOT), "$REPO"),
(config["data_root"], "$DATA"),
(str(report_root), "$REPORT"),
(str(Path.home()), "$HOME"),
]
for reference in config.get("secret_env", {}).values():
value = environ.get(reference, "")
if value:
replacements.append((value, "[REDACTED]"))
def redact(text: str) -> str:
for source, target in sorted(replacements, key=lambda item: len(item[0]), reverse=True):
text = text.replace(source, target).replace(source.replace("\\", "/"), target)
return text
return redact
def _base_result(case_id: str, status: str, reason: str) -> dict[str, Any]:
return {
"schema": SCHEMA,
"case_id": case_id,
"status": status,
"reason": reason,
"assertions": [],
"metrics": {
"peak_rss_bytes": None,
"max_process_count": None,
"denied_access_count": None,
},
"files": [],
"revisions": [],
"command": None,
}
def _has_administrator_permission() -> bool:
if os.name == "nt":
try:
import ctypes
return bool(ctypes.windll.shell32.IsUserAnAdmin())
except (AttributeError, OSError):
return False
return hasattr(os, "geteuid") and os.geteuid() == 0
def _validate_driver_result(case_id: str, result: Any, required_metrics: tuple[str, ...]) -> list[str]:
errors: list[str] = []
if not isinstance(result, dict) or result.get("schema") != SCHEMA:
return ["RESULT_SCHEMA_INVALID"]
if result.get("case_id") != case_id:
errors.append("RESULT_CASE_ID_MISMATCH")
if result.get("status") not in {"PASSED", "FAILED", "SKIPPED", "NOT_APPLICABLE"}:
errors.append("RESULT_STATUS_INVALID")
assertions = result.get("assertions")
if not isinstance(assertions, list) or not assertions:
errors.append("RESULT_ASSERTIONS_MISSING")
elif any(not isinstance(item, dict) or item.get("status") not in {"PASSED", "FAILED"} for item in assertions):
errors.append("RESULT_ASSERTION_INVALID")
metrics = result.get("metrics")
if not isinstance(metrics, dict):
errors.append("RESULT_METRICS_MISSING")
else:
for name in required_metrics:
value = metrics.get(name)
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
errors.append(f"RESULT_METRIC_MISSING:{name}")
for name in ("files", "revisions"):
if not isinstance(result.get(name), list):
errors.append(f"RESULT_{name.upper()}_INVALID")
return errors
def run_case(
case_id: str,
config_path: Path,
config: dict[str, Any],
report_root: Path,
environ: dict[str, str] | None = None,
drivers: dict[str, dict[str, Any]] | None = None,
) -> dict[str, Any]:
environ = dict(os.environ if environ is None else environ)
drivers = CASE_DRIVERS if drivers is None else drivers
started = time.monotonic()
started_utc = _utc_now()
definition = drivers.get(case_id)
if definition is None:
result = _base_result(case_id, "NOT_IMPLEMENTED", "No repository-owned production acceptance driver is registered.")
result.update({"started_utc": started_utc, "finished_utc": _utc_now(), "duration_ms": 0, "exit_code": None})
return result
if definition.get("requires_admin") and not _has_administrator_permission():
result = _base_result(case_id, "BLOCKED", "Administrator permission required by this acceptance case is unavailable.")
result.update({"started_utc": started_utc, "finished_utc": _utc_now(), "duration_ms": 0, "exit_code": None})
return result
required_platforms = tuple(definition.get("platform_profiles", ()))
if required_platforms and config["platform_profile"] not in required_platforms:
result = _base_result(case_id, "BLOCKED", "Configured platform profile is not supported by this acceptance driver.")
result["supported_platform_profiles"] = list(required_platforms)
result.update({"started_utc": started_utc, "finished_utc": _utc_now(), "duration_ms": 0, "exit_code": None})
return result
driver = (ROOT / str(definition["driver"])).resolve()
if not _inside(driver, ROOT) or not driver.is_file():
result = _base_result(case_id, "BLOCKED", "Registered acceptance driver is missing or outside the repository.")
result.update({"started_utc": started_utc, "finished_utc": _utc_now(), "duration_ms": 0, "exit_code": None})
return result
missing = [name for name in definition.get("required_artifacts", ()) if not Path(config.get("artifacts", {}).get(name, "")).is_file()]
if missing:
result = _base_result(case_id, "BLOCKED", "Required artifacts are missing.")
result["missing_artifacts"] = missing
result.update({"started_utc": started_utc, "finished_utc": _utc_now(), "duration_ms": 0, "exit_code": None})
return result
missing_secrets = [name for name in definition.get("required_secret_env", ()) if name not in config.get("secret_env", {})]
if missing_secrets:
result = _base_result(case_id, "BLOCKED", "Required secret environment references are missing.")
result["missing_secret_env"] = missing_secrets
result.update({"started_utc": started_utc, "finished_utc": _utc_now(), "duration_ms": 0, "exit_code": None})
return result
result_path = report_root / "driver-results" / f"{case_id}.json"
result_path.parent.mkdir(parents=True, exist_ok=True)
command = [sys.executable, str(driver), "--config", str(config_path), "--output", str(result_path)]
child_env = dict(environ)
child_env.update({
"OPENNEXUS_ACCEPTANCE_CASE_ID": case_id,
"OPENNEXUS_ACCEPTANCE_DATA_ROOT": config["data_root"],
"OPENNEXUS_ACCEPTANCE_REPORT_ROOT": str(report_root),
})
timed_out = False
try:
completed = subprocess.run(
command,
cwd=ROOT,
env=child_env,
capture_output=True,
text=True,
timeout=int(definition.get("timeout_seconds", 1800)),
check=False,
)
exit_code = completed.returncode
output = completed.stdout + ("\n" if completed.stdout and completed.stderr else "") + completed.stderr
except subprocess.TimeoutExpired as error:
timed_out = True
exit_code = None
output = (error.stdout or "") + (error.stderr or "")
if isinstance(output, bytes):
output = output.decode("utf-8", errors="replace")
redact = _redactor(config, report_root, environ)
encoded = redact(output).encode("utf-8", errors="replace")[:MAX_LOG_BYTES]
(report_root / "logs").mkdir(exist_ok=True)
(report_root / "logs" / f"{case_id}.log").write_bytes(encoded)
if timed_out:
result = _base_result(case_id, "FAILED", "Acceptance driver timed out.")
elif not result_path.is_file():
result = _base_result(case_id, "FAILED", "Acceptance driver did not produce a result.")
else:
try:
result = json.loads(result_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
result = _base_result(case_id, "FAILED", "Acceptance driver result is unreadable.")
errors = _validate_driver_result(case_id, result, tuple(definition.get("required_metrics", ())))
if not isinstance(result, dict):
result = _base_result(case_id, "FAILED", "Acceptance driver result must be a JSON object.")
if errors or exit_code != 0 or result.get("status") != "PASSED":
if result.get("status") == "PASSED":
result["status"] = "FAILED"
result["runner_errors"] = errors + ([] if exit_code in {0, None} else ["DRIVER_EXIT_NONZERO"])
serialized = json.dumps(result, ensure_ascii=False)
leaked = [reference for reference in config.get("secret_env", {}).values() if environ.get(reference) and environ[reference] in serialized]
if leaked:
result = _base_result(case_id, "FAILED", "Acceptance result contained configured secret material.")
result["runner_errors"] = ["RESULT_SECRET_LEAK"]
result.update({
"started_utc": started_utc,
"finished_utc": _utc_now(),
"duration_ms": round((time.monotonic() - started) * 1000),
"exit_code": exit_code,
"log": f"logs/{case_id}.log",
"command": [
"python", str(definition["driver"]), "--config", "$CONFIG",
"--output", f"$REPORT/driver-results/{case_id}.json",
],
})
return result
def _write_junit(path: Path, results: list[dict[str, Any]], elapsed: float) -> None:
failures = sum(result["status"] != "PASSED" for result in results)
suite = ElementTree.Element("testsuite", {
"name": "OpenNexus phase3 production acceptance",
"tests": str(len(results)),
"failures": str(failures),
"errors": "0",
"skipped": "0",
"time": f"{elapsed:.3f}",
})
for result in results:
case = ElementTree.SubElement(suite, "testcase", {
"classname": "phase3.production",
"name": result["case_id"],
"time": f"{result.get('duration_ms', 0) / 1000:.3f}",
})
if result["status"] != "PASSED":
failure = ElementTree.SubElement(case, "failure", {"type": result["status"]})
failure.text = str(result.get("reason") or ",".join(result.get("runner_errors", ())))
ElementTree.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True)
def _repository_evidence(config: dict[str, Any]) -> dict[str, Any]:
locks = {}
for relative in ("backend/uv.lock", "frontend/pnpm-lock.yaml", "frontend/src-tauri/Cargo.lock", "server sync/uv.lock", "community-server/uv.lock"):
path = ROOT / relative
if path.is_file():
locks[relative] = _sha256(path)
artifacts = {}
for name, raw in config.get("artifacts", {}).items():
path = Path(raw)
artifacts[name] = _sha256(path) if path.is_file() else None
commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, text=True, check=False).stdout.strip()
return {"commit": commit or None, "lock_sha256": locks, "artifact_sha256": artifacts}
def execute(args: argparse.Namespace, environ: dict[str, str] | None = None) -> int:
environ = os.environ if environ is None else environ
if args.list_cases:
payload = {suite: list(cases) for suite, cases in CASE_SUITES.items()}
print(json.dumps(payload, ensure_ascii=False, indent=2) if args.json else "\n".join(f"{suite}: {', '.join(cases)}" for suite, cases in CASE_SUITES.items()))
return 0
if not args.suite or not args.config or not args.report_dir:
raise AcceptanceError("ARGUMENTS_REQUIRED")
selected = select_cases(args.suite, args.case)
config_path = Path(args.config).resolve()
config = load_config(config_path, environ)
if args.suite == "all" and not config.get("platform_profile"):
raise AcceptanceError("CONFIG_PLATFORM_PROFILE_REQUIRED")
prepare_isolated_root(config)
report_root = Path(args.report_dir).resolve(strict=False)
if _inside(report_root, VAULT_ROOT) or _inside(VAULT_ROOT, report_root):
raise AcceptanceError("REPORT_PERSONAL_VAULT_FORBIDDEN")
if report_root.exists() and (report_root.is_symlink() or not report_root.is_dir()):
raise AcceptanceError("REPORT_DIRECTORY_INVALID")
if report_root.exists() and any(report_root.iterdir()):
raise AcceptanceError("REPORT_DIRECTORY_NOT_EMPTY")
report_root.mkdir(parents=True, exist_ok=True)
started = time.monotonic()
results = [run_case(case_id, config_path, config, report_root, environ) for case_id in selected]
cases_root = report_root / "cases"
cases_root.mkdir()
for result in results:
(cases_root / f"{result['case_id']}.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
elapsed = time.monotonic() - started
_write_junit(report_root / "junit.xml", results, elapsed)
manifest = {"schema": SCHEMA, "suites": {name: list(cases) for name, cases in CASE_SUITES.items()}, "all_cases": list(ALL_CASES)}
if len(ALL_CASES) != 30 or len(set(ALL_CASES)) != 30:
raise AcceptanceError("CASE_MANIFEST_INCOMPLETE")
(report_root / "case-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
passed = sum(result["status"] == "PASSED" for result in results)
summary = {
"schema": SCHEMA,
"run_id": config["run_id"],
"suite": args.suite,
"platform_profile": config["platform_profile"],
"status": "PASSED" if passed == len(results) else "NOT_PASSED",
"selected": list(selected),
"passed": passed,
"failed": len(results) - passed,
"duration_ms": round(elapsed * 1000),
"generated_utc": _utc_now(),
"config_sha256": _sha256(config_path),
"contains_credentials": False,
**_repository_evidence(config),
}
(report_root / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"status": summary["status"], "report_dir": str(report_root), "passed": passed, "total": len(results)}, ensure_ascii=False))
return 0 if summary["status"] == "PASSED" else 1
def parser() -> argparse.ArgumentParser:
result = argparse.ArgumentParser(description=__doc__)
result.add_argument("--suite", choices=(*CASE_SUITES, "all"))
result.add_argument("--case")
result.add_argument("--config")
result.add_argument("--report-dir")
result.add_argument("--list-cases", action="store_true")
result.add_argument("--json", action="store_true", help="Use JSON with --list-cases")
return result
def main(argv: list[str] | None = None) -> int:
try:
return execute(parser().parse_args(argv))
except AcceptanceError as error:
print(str(error), file=sys.stderr)
return 2
if __name__ == "__main__":
raise SystemExit(main())