fix(sync): 依赖超时后限制就绪检查工作量
This commit is contained in:
@@ -19,6 +19,7 @@ from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from .database import Database, password_hash, row, rows, run
|
||||
from .models import Commit, Login, Refresh, Upload, VaultCreate
|
||||
from .readiness import Readiness
|
||||
|
||||
|
||||
class SyncError(Exception):
|
||||
@@ -116,22 +117,13 @@ def create_app(db: Database, objects, staging: Path, *, quota=1024**3, clock=tim
|
||||
finally:
|
||||
objects.delete(key)
|
||||
|
||||
ready_lock = asyncio.Lock()
|
||||
ready_cache = {"until": 0.0, "ok": False}
|
||||
readiness = Readiness(readiness_probe)
|
||||
|
||||
@app.get("/ready")
|
||||
async def ready():
|
||||
async with ready_lock:
|
||||
if time.monotonic() >= ready_cache["until"]:
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.to_thread(readiness_probe), timeout=3)
|
||||
ready_cache["ok"] = True
|
||||
except Exception:
|
||||
ready_cache["ok"] = False
|
||||
ready_cache["until"] = time.monotonic() + 5
|
||||
if not ready_cache["ok"]:
|
||||
raise SyncError(503, "DEPENDENCY_UNAVAILABLE")
|
||||
return {"status": "ready", "schema": 1}
|
||||
if not await readiness.check():
|
||||
raise SyncError(503, "DEPENDENCY_UNAVAILABLE")
|
||||
return {"status": "ready", "schema": 1}
|
||||
|
||||
@app.get("/sync/v1/handshake")
|
||||
def handshake(protocol: int = 1):
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Bound readiness work even when a synchronous dependency ignores its timeout."""
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
|
||||
class Readiness:
|
||||
def __init__(self, probe, *, timeout=3, cache_seconds=5):
|
||||
self.probe = probe
|
||||
self.timeout = timeout
|
||||
self.cache_seconds = cache_seconds
|
||||
self.lock = asyncio.Lock()
|
||||
self.running = None
|
||||
self.until = 0.0
|
||||
self.ok = False
|
||||
|
||||
@staticmethod
|
||||
def consume(task):
|
||||
# A request can time out or disconnect before the synchronous probe ends.
|
||||
# Retrieve late exceptions without logging dependency messages/secrets.
|
||||
if not task.cancelled():
|
||||
task.exception()
|
||||
|
||||
async def check(self):
|
||||
async with self.lock:
|
||||
if time.monotonic() < self.until:
|
||||
return self.ok
|
||||
if self.running is None or self.running.done():
|
||||
self.running = asyncio.create_task(asyncio.to_thread(self.probe))
|
||||
self.running.add_done_callback(self.consume)
|
||||
try:
|
||||
# Cancelling a to_thread await does not stop its OS thread. Keep
|
||||
# the task alive so subsequent requests reuse the same probe.
|
||||
await asyncio.wait_for(asyncio.shield(self.running), self.timeout)
|
||||
self.ok = True
|
||||
except Exception:
|
||||
self.ok = False
|
||||
self.until = time.monotonic() + self.cache_seconds
|
||||
return self.ok
|
||||
Reference in New Issue
Block a user