docs/development/OpenNexus生产化实施进度-2026-09-08.md

@@ -661,3 +661,13 @@ Core 的独立数据目录目前不等于已授权 Vault。Python 旧笔记写
 - 实际运行 Uvicorn TCP 回环测试：四个 1 MiB + 17 字节验证非整块末尾；四个 100 MiB 验证实际大对象。两项通过，耗时 16.28 秒；大对象上传至下载/隔离验证全部完成约 9.31 秒，400 MiB 内容全部摘要一致。日志 .build/sync-upload-benchmark.log，JUnit .build/sync-upload-benchmark.xml。
 - 此结果是同一 Python 进程内 SQLite/DiskObjects 单服务实例与 HTTP 客户端的受控检查，不是 PostgreSQL/MinIO 两 worker 基准。service_rss_bytes 明确 null，acceptance 明确 NOT_ASSESSED，不能将传输通过当作 RSS≤2 GiB 或完整 S-09 通过。本机 Get-Command docker 未找到可执行文件；生产拓扑/服务内存采集/30 分钟负载仍需继续。
 - 最终 Sync 服务端全套 27 通过、2 项依赖弃用警告，22.39 秒；日志 .build/sync-server-four-upload-full.log，JUnit .build/sync-server-four-upload-full.xml。用户 Vault 修改保持原状，完整生产化目标未完成。
+
+
+## 增量：就绪探测超时后的工作线程有界性
+
+- 只读复查测试服务器：/health 返回 200，/ready 返回 503 DEPENDENCY_UNAVAILABLE；SSH 22 TCP 可连接但读取 banner 得到 0 字节，尚未进入认证。没有远端写入；不能据此定位具体依赖或宣称部署已恢复。
+- readiness 原先 wait_for(to_thread(...), 3) 超时后取消包装任务，但不能停止底层同步线程；缓存到期后可能再创建新的阻塞探测。本轮提取 Readiness，使用 shield 保持任务可追踪，并在其结束前复用同一进行中的任务，每个应用 worker 最多一个此类依赖探测。保持 3 秒等待、5 秒结果缓存与对外固定错误码。
+- 请求取消不丢失后台任务引用；晚到异常由完成回调检索，不写入依赖错误文本或凭据。完成后的后续新探测可恢复 ready。测试覆盖取消后 20 个探测请求均失败但底层调用数仍为 1、释放阻塞后异常收尾、健康新探测，以及失败/成功结果缓存。
+- 本改动限制同时进行的后台探测数量，不能强杀陷入 OS/驱动 I/O 的线程，不作为异常退出时限证明。真实依赖超时、远端恢复和服务负载验收仍未完成。
+- 首次全量测试暴露提取代码时误删相邻 handshake 路由（1 failed / 28 passed）；差异审查已恢复原路由，此失败不计为最终通过证据，最终结果另列。
+- 修正后最终 Sync 全套 29 通过、2 项依赖弃用警告，22.83 秒，包含实际四并发 100 MiB TCP 传输。日志 .build/sync-readiness-full-final.log，JUnit .build/sync-readiness-full-final.xml；完整生产化目标仍未完成。

server sync/sync_server/app.py

@@ -19,6 +19,7 @@ from starlette.concurrency import run_in_threadpool
 
 from .database import Database, password_hash, row, rows, run
 from .models import Commit, Login, Refresh, Upload, VaultCreate
+from .readiness import Readiness
 
 
 class SyncError(Exception):
@@ -116,22 +117,13 @@ def create_app(db: Database, objects, staging: Path, *, quota=1024**3, clock=tim
         finally:
             objects.delete(key)
 
-    ready_lock = asyncio.Lock()
-    ready_cache = {"until": 0.0, "ok": False}
+    readiness = Readiness(readiness_probe)
 
     @app.get("/ready")
     async def ready():
-        async with ready_lock:
-            if time.monotonic() >= ready_cache["until"]:
-                try:
-                    await asyncio.wait_for(asyncio.to_thread(readiness_probe), timeout=3)
-                    ready_cache["ok"] = True
-                except Exception:
-                    ready_cache["ok"] = False
-                ready_cache["until"] = time.monotonic() + 5
-            if not ready_cache["ok"]:
-                raise SyncError(503, "DEPENDENCY_UNAVAILABLE")
-            return {"status": "ready", "schema": 1}
+        if not await readiness.check():
+            raise SyncError(503, "DEPENDENCY_UNAVAILABLE")
+        return {"status": "ready", "schema": 1}
 
     @app.get("/sync/v1/handshake")
     def handshake(protocol: int = 1):

server sync/sync_server/readiness.py

@@ -0,0 +1,38 @@
+"""Bound readiness work even when a synchronous dependency ignores its timeout."""
+import asyncio
+import time
+
+
+class Readiness:
+    def __init__(self, probe, *, timeout=3, cache_seconds=5):
+        self.probe = probe
+        self.timeout = timeout
+        self.cache_seconds = cache_seconds
+        self.lock = asyncio.Lock()
+        self.running = None
+        self.until = 0.0
+        self.ok = False
+
+    @staticmethod
+    def consume(task):
+        # A request can time out or disconnect before the synchronous probe ends.
+        # Retrieve late exceptions without logging dependency messages/secrets.
+        if not task.cancelled():
+            task.exception()
+
+    async def check(self):
+        async with self.lock:
+            if time.monotonic() < self.until:
+                return self.ok
+            if self.running is None or self.running.done():
+                self.running = asyncio.create_task(asyncio.to_thread(self.probe))
+                self.running.add_done_callback(self.consume)
+            try:
+                # Cancelling a to_thread await does not stop its OS thread. Keep
+                # the task alive so subsequent requests reuse the same probe.
+                await asyncio.wait_for(asyncio.shield(self.running), self.timeout)
+                self.ok = True
+            except Exception:
+                self.ok = False
+            self.until = time.monotonic() + self.cache_seconds
+            return self.ok

server sync/tests/test_readiness.py

@@ -0,0 +1,56 @@
+"""Bounded worker use, request cancellation, cached failures and recovery."""
+import asyncio
+from threading import Event
+
+from sync_server.readiness import Readiness
+
+
+def test_timeout_and_cancel_never_spawn_overlapping_dependency_probes():
+    entered, release = Event(), Event()
+    calls = []
+
+    def slow():
+        calls.append(1)
+        entered.set()
+        if not release.wait(10):
+            raise TimeoutError("fixture was not released")
+        raise OSError("late dependency exception must be consumed")
+
+    async def run():
+        ready = Readiness(slow, timeout=.01, cache_seconds=0)
+        request = asyncio.create_task(ready.check())
+        try:
+            while not entered.is_set():
+                await asyncio.sleep(.001)
+            request.cancel()
+            await asyncio.gather(request, return_exceptions=True)
+            assert ready.running is not None and not ready.running.done()
+            assert await asyncio.gather(*(ready.check() for _ in range(20))) == [False] * 20
+            assert len(calls) == 1
+        finally:
+            release.set()
+            await asyncio.gather(ready.running, return_exceptions=True)
+        # A completed failed probe does not prevent a new healthy attempt.
+        ready.probe = lambda: None
+        assert await ready.check() is True
+    asyncio.run(run())
+
+
+def test_success_and_failure_cache_then_refresh():
+    calls = []
+
+    def probe():
+        calls.append(1)
+        if len(calls) == 1:
+            raise OSError("controlled dependency outage")
+
+    async def run():
+        ready = Readiness(probe, cache_seconds=.02)
+        assert await ready.check() is False
+        assert await asyncio.gather(*(ready.check() for _ in range(20))) == [False] * 20
+        assert len(calls) == 1
+        await asyncio.sleep(.03)
+        assert await ready.check() is True
+        assert await ready.check() is True
+        assert len(calls) == 2
+    asyncio.run(run())