fix(sync): 依赖超时后限制就绪检查工作量

server sync/sync_server/app.py

@@ -19,6 +19,7 @@ from starlette.concurrency import run_in_threadpool
 
 from .database import Database, password_hash, row, rows, run
 from .models import Commit, Login, Refresh, Upload, VaultCreate
+from .readiness import Readiness
 
 
 class SyncError(Exception):
@@ -116,22 +117,13 @@ def create_app(db: Database, objects, staging: Path, *, quota=1024**3, clock=tim
         finally:
             objects.delete(key)
 
-    ready_lock = asyncio.Lock()
+    readiness = Readiness(readiness_probe)
-    ready_cache = {"until": 0.0, "ok": False}
 
     @app.get("/ready")
     async def ready():
-        async with ready_lock:
+        if not await readiness.check():
-            if time.monotonic() >= ready_cache["until"]:
+            raise SyncError(503, "DEPENDENCY_UNAVAILABLE")
-                try:
+        return {"status": "ready", "schema": 1}
-                    await asyncio.wait_for(asyncio.to_thread(readiness_probe), timeout=3)
-                    ready_cache["ok"] = True
-                except Exception:
-                    ready_cache["ok"] = False
-                ready_cache["until"] = time.monotonic() + 5
-            if not ready_cache["ok"]:
-                raise SyncError(503, "DEPENDENCY_UNAVAILABLE")
-            return {"status": "ready", "schema": 1}
 
     @app.get("/sync/v1/handshake")
     def handshake(protocol: int = 1):

server sync/sync_server/readiness.py

@@ -0,0 +1,38 @@
+"""Bound readiness work even when a synchronous dependency ignores its timeout."""
+import asyncio
+import time
+
+
+class Readiness:
+    def __init__(self, probe, *, timeout=3, cache_seconds=5):
+        self.probe = probe
+        self.timeout = timeout
+        self.cache_seconds = cache_seconds
+        self.lock = asyncio.Lock()
+        self.running = None
+        self.until = 0.0
+        self.ok = False
+
+    @staticmethod
+    def consume(task):
+        # A request can time out or disconnect before the synchronous probe ends.
+        # Retrieve late exceptions without logging dependency messages/secrets.
+        if not task.cancelled():
+            task.exception()
+
+    async def check(self):
+        async with self.lock:
+            if time.monotonic() < self.until:
+                return self.ok
+            if self.running is None or self.running.done():
+                self.running = asyncio.create_task(asyncio.to_thread(self.probe))
+                self.running.add_done_callback(self.consume)
+            try:
+                # Cancelling a to_thread await does not stop its OS thread. Keep
+                # the task alive so subsequent requests reuse the same probe.
+                await asyncio.wait_for(asyncio.shield(self.running), self.timeout)
+                self.ok = True
+            except Exception:
+                self.ok = False
+            self.until = time.monotonic() + self.cache_seconds
+            return self.ok

server sync/tests/test_readiness.py

@@ -0,0 +1,56 @@
+"""Bounded worker use, request cancellation, cached failures and recovery."""
+import asyncio
+from threading import Event
+
+from sync_server.readiness import Readiness
+
+
+def test_timeout_and_cancel_never_spawn_overlapping_dependency_probes():
+    entered, release = Event(), Event()
+    calls = []
+
+    def slow():
+        calls.append(1)
+        entered.set()
+        if not release.wait(10):
+            raise TimeoutError("fixture was not released")
+        raise OSError("late dependency exception must be consumed")
+
+    async def run():
+        ready = Readiness(slow, timeout=.01, cache_seconds=0)
+        request = asyncio.create_task(ready.check())
+        try:
+            while not entered.is_set():
+                await asyncio.sleep(.001)
+            request.cancel()
+            await asyncio.gather(request, return_exceptions=True)
+            assert ready.running is not None and not ready.running.done()
+            assert await asyncio.gather(*(ready.check() for _ in range(20))) == [False] * 20
+            assert len(calls) == 1
+        finally:
+            release.set()
+            await asyncio.gather(ready.running, return_exceptions=True)
+        # A completed failed probe does not prevent a new healthy attempt.
+        ready.probe = lambda: None
+        assert await ready.check() is True
+    asyncio.run(run())
+
+
+def test_success_and_failure_cache_then_refresh():
+    calls = []
+
+    def probe():
+        calls.append(1)
+        if len(calls) == 1:
+            raise OSError("controlled dependency outage")
+
+    async def run():
+        ready = Readiness(probe, cache_seconds=.02)
+        assert await ready.check() is False
+        assert await asyncio.gather(*(ready.check() for _ in range(20))) == [False] * 20
+        assert len(calls) == 1
+        await asyncio.sleep(.03)
+        assert await ready.check() is True
+        assert await ready.check() is True
+        assert len(calls) == 2
+    asyncio.run(run())