diff --git a/server sync/sync_server/app.py b/server sync/sync_server/app.py index a324815..df4e17b 100644 --- a/server sync/sync_server/app.py +++ b/server sync/sync_server/app.py @@ -19,6 +19,7 @@ from starlette.concurrency import run_in_threadpool from .database import Database, password_hash, row, rows, run from .models import Commit, Login, Refresh, Upload, VaultCreate +from .readiness import Readiness class SyncError(Exception): @@ -116,22 +117,13 @@ def create_app(db: Database, objects, staging: Path, *, quota=1024**3, clock=tim finally: objects.delete(key) - ready_lock = asyncio.Lock() - ready_cache = {"until": 0.0, "ok": False} + readiness = Readiness(readiness_probe) @app.get("/ready") async def ready(): - async with ready_lock: - if time.monotonic() >= ready_cache["until"]: - try: - await asyncio.wait_for(asyncio.to_thread(readiness_probe), timeout=3) - ready_cache["ok"] = True - except Exception: - ready_cache["ok"] = False - ready_cache["until"] = time.monotonic() + 5 - if not ready_cache["ok"]: - raise SyncError(503, "DEPENDENCY_UNAVAILABLE") - return {"status": "ready", "schema": 1} + if not await readiness.check(): + raise SyncError(503, "DEPENDENCY_UNAVAILABLE") + return {"status": "ready", "schema": 1} @app.get("/sync/v1/handshake") def handshake(protocol: int = 1): diff --git a/server sync/sync_server/readiness.py b/server sync/sync_server/readiness.py new file mode 100644 index 0000000..e6f5a83 --- /dev/null +++ b/server sync/sync_server/readiness.py @@ -0,0 +1,38 @@ +"""Bound readiness work even when a synchronous dependency ignores its timeout.""" +import asyncio +import time + + +class Readiness: + def __init__(self, probe, *, timeout=3, cache_seconds=5): + self.probe = probe + self.timeout = timeout + self.cache_seconds = cache_seconds + self.lock = asyncio.Lock() + self.running = None + self.until = 0.0 + self.ok = False + + @staticmethod + def consume(task): + # A request can time out or disconnect before the synchronous probe ends. + # Retrieve late exceptions without logging dependency messages/secrets. + if not task.cancelled(): + task.exception() + + async def check(self): + async with self.lock: + if time.monotonic() < self.until: + return self.ok + if self.running is None or self.running.done(): + self.running = asyncio.create_task(asyncio.to_thread(self.probe)) + self.running.add_done_callback(self.consume) + try: + # Cancelling a to_thread await does not stop its OS thread. Keep + # the task alive so subsequent requests reuse the same probe. + await asyncio.wait_for(asyncio.shield(self.running), self.timeout) + self.ok = True + except Exception: + self.ok = False + self.until = time.monotonic() + self.cache_seconds + return self.ok diff --git a/server sync/tests/test_readiness.py b/server sync/tests/test_readiness.py new file mode 100644 index 0000000..3ffb1ea --- /dev/null +++ b/server sync/tests/test_readiness.py @@ -0,0 +1,56 @@ +"""Bounded worker use, request cancellation, cached failures and recovery.""" +import asyncio +from threading import Event + +from sync_server.readiness import Readiness + + +def test_timeout_and_cancel_never_spawn_overlapping_dependency_probes(): + entered, release = Event(), Event() + calls = [] + + def slow(): + calls.append(1) + entered.set() + if not release.wait(10): + raise TimeoutError("fixture was not released") + raise OSError("late dependency exception must be consumed") + + async def run(): + ready = Readiness(slow, timeout=.01, cache_seconds=0) + request = asyncio.create_task(ready.check()) + try: + while not entered.is_set(): + await asyncio.sleep(.001) + request.cancel() + await asyncio.gather(request, return_exceptions=True) + assert ready.running is not None and not ready.running.done() + assert await asyncio.gather(*(ready.check() for _ in range(20))) == [False] * 20 + assert len(calls) == 1 + finally: + release.set() + await asyncio.gather(ready.running, return_exceptions=True) + # A completed failed probe does not prevent a new healthy attempt. + ready.probe = lambda: None + assert await ready.check() is True + asyncio.run(run()) + + +def test_success_and_failure_cache_then_refresh(): + calls = [] + + def probe(): + calls.append(1) + if len(calls) == 1: + raise OSError("controlled dependency outage") + + async def run(): + ready = Readiness(probe, cache_seconds=.02) + assert await ready.check() is False + assert await asyncio.gather(*(ready.check() for _ in range(20))) == [False] * 20 + assert len(calls) == 1 + await asyncio.sleep(.03) + assert await ready.check() is True + assert await ready.check() is True + assert len(calls) == 2 + asyncio.run(run())