diff --git a/docs/development/OpenNexus生产化实施进度-2026-09-08.md b/docs/development/OpenNexus生产化实施进度-2026-09-08.md index e651f21..2b5119e 100644 --- a/docs/development/OpenNexus生产化实施进度-2026-09-08.md +++ b/docs/development/OpenNexus生产化实施进度-2026-09-08.md @@ -661,3 +661,13 @@ Core 的独立数据目录目前不等于已授权 Vault。Python 旧笔记写 - 实际运行 Uvicorn TCP 回环测试：四个 1 MiB + 17 字节验证非整块末尾；四个 100 MiB 验证实际大对象。两项通过，耗时 16.28 秒；大对象上传至下载/隔离验证全部完成约 9.31 秒，400 MiB 内容全部摘要一致。日志 .build/sync-upload-benchmark.log，JUnit .build/sync-upload-benchmark.xml。 - 此结果是同一 Python 进程内 SQLite/DiskObjects 单服务实例与 HTTP 客户端的受控检查，不是 PostgreSQL/MinIO 两 worker 基准。service_rss_bytes 明确 null，acceptance 明确 NOT_ASSESSED，不能将传输通过当作 RSS≤2 GiB 或完整 S-09 通过。本机 Get-Command docker 未找到可执行文件；生产拓扑/服务内存采集/30 分钟负载仍需继续。 - 最终 Sync 服务端全套 27 通过、2 项依赖弃用警告，22.39 秒；日志 .build/sync-server-four-upload-full.log，JUnit .build/sync-server-four-upload-full.xml。用户 Vault 修改保持原状，完整生产化目标未完成。 + + +## 增量：就绪探测超时后的工作线程有界性 + +- 只读复查测试服务器：/health 返回 200，/ready 返回 503 DEPENDENCY_UNAVAILABLE；SSH 22 TCP 可连接但读取 banner 得到 0 字节，尚未进入认证。没有远端写入；不能据此定位具体依赖或宣称部署已恢复。 +- readiness 原先 wait_for(to_thread(...), 3) 超时后取消包装任务，但不能停止底层同步线程；缓存到期后可能再创建新的阻塞探测。本轮提取 Readiness，使用 shield 保持任务可追踪，并在其结束前复用同一进行中的任务，每个应用 worker 最多一个此类依赖探测。保持 3 秒等待、5 秒结果缓存与对外固定错误码。 +- 请求取消不丢失后台任务引用；晚到异常由完成回调检索，不写入依赖错误文本或凭据。完成后的后续新探测可恢复 ready。测试覆盖取消后 20 个探测请求均失败但底层调用数仍为 1、释放阻塞后异常收尾、健康新探测，以及失败/成功结果缓存。 +- 本改动限制同时进行的后台探测数量，不能强杀陷入 OS/驱动 I/O 的线程，不作为异常退出时限证明。真实依赖超时、远端恢复和服务负载验收仍未完成。 +- 首次全量测试暴露提取代码时误删相邻 handshake 路由（1 failed / 28 passed）；差异审查已恢复原路由，此失败不计为最终通过证据，最终结果另列。 +- 修正后最终 Sync 全套 29 通过、2 项依赖弃用警告，22.83 秒，包含实际四并发 100 MiB TCP 传输。日志 .build/sync-readiness-full-final.log，JUnit .build/sync-readiness-full-final.xml；完整生产化目标仍未完成。 diff --git a/server sync/sync_server/app.py b/server sync/sync_server/app.py index a324815..df4e17b 100644 --- a/server sync/sync_server/app.py +++ b/server sync/sync_server/app.py @@ -19,6 +19,7 @@ from starlette.concurrency import run_in_threadpool from .database import Database, password_hash, row, rows, run from .models import Commit, Login, Refresh, Upload, VaultCreate +from .readiness import Readiness class SyncError(Exception): @@ -116,22 +117,13 @@ def create_app(db: Database, objects, staging: Path, *, quota=1024**3, clock=tim finally: objects.delete(key) - ready_lock = asyncio.Lock() - ready_cache = {"until": 0.0, "ok": False} + readiness = Readiness(readiness_probe) @app.get("/ready") async def ready(): - async with ready_lock: - if time.monotonic() >= ready_cache["until"]: - try: - await asyncio.wait_for(asyncio.to_thread(readiness_probe), timeout=3) - ready_cache["ok"] = True - except Exception: - ready_cache["ok"] = False - ready_cache["until"] = time.monotonic() + 5 - if not ready_cache["ok"]: - raise SyncError(503, "DEPENDENCY_UNAVAILABLE") - return {"status": "ready", "schema": 1} + if not await readiness.check(): + raise SyncError(503, "DEPENDENCY_UNAVAILABLE") + return {"status": "ready", "schema": 1} @app.get("/sync/v1/handshake") def handshake(protocol: int = 1): diff --git a/server sync/sync_server/readiness.py b/server sync/sync_server/readiness.py new file mode 100644 index 0000000..e6f5a83 --- /dev/null +++ b/server sync/sync_server/readiness.py @@ -0,0 +1,38 @@ +"""Bound readiness work even when a synchronous dependency ignores its timeout.""" +import asyncio +import time + + +class Readiness: + def __init__(self, probe, *, timeout=3, cache_seconds=5): + self.probe = probe + self.timeout = timeout + self.cache_seconds = cache_seconds + self.lock = asyncio.Lock() + self.running = None + self.until = 0.0 + self.ok = False + + @staticmethod + def consume(task): + # A request can time out or disconnect before the synchronous probe ends. + # Retrieve late exceptions without logging dependency messages/secrets. + if not task.cancelled(): + task.exception() + + async def check(self): + async with self.lock: + if time.monotonic() < self.until: + return self.ok + if self.running is None or self.running.done(): + self.running = asyncio.create_task(asyncio.to_thread(self.probe)) + self.running.add_done_callback(self.consume) + try: + # Cancelling a to_thread await does not stop its OS thread. Keep + # the task alive so subsequent requests reuse the same probe. + await asyncio.wait_for(asyncio.shield(self.running), self.timeout) + self.ok = True + except Exception: + self.ok = False + self.until = time.monotonic() + self.cache_seconds + return self.ok diff --git a/server sync/tests/test_readiness.py b/server sync/tests/test_readiness.py new file mode 100644 index 0000000..3ffb1ea --- /dev/null +++ b/server sync/tests/test_readiness.py @@ -0,0 +1,56 @@ +"""Bounded worker use, request cancellation, cached failures and recovery.""" +import asyncio +from threading import Event + +from sync_server.readiness import Readiness + + +def test_timeout_and_cancel_never_spawn_overlapping_dependency_probes(): + entered, release = Event(), Event() + calls = [] + + def slow(): + calls.append(1) + entered.set() + if not release.wait(10): + raise TimeoutError("fixture was not released") + raise OSError("late dependency exception must be consumed") + + async def run(): + ready = Readiness(slow, timeout=.01, cache_seconds=0) + request = asyncio.create_task(ready.check()) + try: + while not entered.is_set(): + await asyncio.sleep(.001) + request.cancel() + await asyncio.gather(request, return_exceptions=True) + assert ready.running is not None and not ready.running.done() + assert await asyncio.gather(*(ready.check() for _ in range(20))) == [False] * 20 + assert len(calls) == 1 + finally: + release.set() + await asyncio.gather(ready.running, return_exceptions=True) + # A completed failed probe does not prevent a new healthy attempt. + ready.probe = lambda: None + assert await ready.check() is True + asyncio.run(run()) + + +def test_success_and_failure_cache_then_refresh(): + calls = [] + + def probe(): + calls.append(1) + if len(calls) == 1: + raise OSError("controlled dependency outage") + + async def run(): + ready = Readiness(probe, cache_seconds=.02) + assert await ready.check() is False + assert await asyncio.gather(*(ready.check() for _ in range(20))) == [False] * 20 + assert len(calls) == 1 + await asyncio.sleep(.03) + assert await ready.check() is True + assert await ready.check() is True + assert len(calls) == 2 + asyncio.run(run())