From 70731173f49102fb5eb450e10b4a09aca4f938ce Mon Sep 17 00:00:00 2001 From: KiriAky 107 Date: Wed, 9 Sep 2026 06:36:42 +0800 Subject: [PATCH] =?UTF-8?q?fix(sync):=20=E5=B0=86=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E6=8C=81=E4=B9=85=E5=8C=96=E7=A7=BB=E5=87=BA=20ASGI=20?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E5=BE=AA=E7=8E=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server sync/sync_server/app.py | 11 +++- server sync/tests/test_production_storage.py | 63 ++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/server sync/sync_server/app.py b/server sync/sync_server/app.py index a07a22a..a324815 100644 --- a/server sync/sync_server/app.py +++ b/server sync/sync_server/app.py @@ -15,6 +15,7 @@ from fastapi import FastAPI, Header, Query, Request from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse, FileResponse from starlette.background import BackgroundTask +from starlette.concurrency import run_in_threadpool from .database import Database, password_hash, row, rows, run from .models import Commit, Login, Refresh, Upload, VaultCreate @@ -257,9 +258,15 @@ def create_app(db: Database, objects, staging: Path, *, quota=1024**3, clock=tim offset: int = Query(ge=0), authorization: str = Header(default="")): data = bytearray() async for chunk in request.stream(): - data.extend(chunk) - if len(data) > 1048576: + if len(chunk) > 1048576 - len(data): raise SyncError(413, "CHUNK_TOO_LARGE") + data.extend(chunk) + # Keep the transaction and durable write on one worker thread. A slow + # database lock or fsync must not block this worker's ASGI event loop. + return await run_in_threadpool(persist_upload_chunk, vault_id, upload_id, + authorization, offset, data) + + def persist_upload_chunk(vault_id, upload_id, authorization, offset, data): with db.transaction() as conn: upload = authorized_upload(conn, vault_id, upload_id, authorization) reconcile_staging(upload) diff --git a/server sync/tests/test_production_storage.py b/server sync/tests/test_production_storage.py index a60825e..f86919a 100644 --- a/server sync/tests/test_production_storage.py +++ b/server sync/tests/test_production_storage.py @@ -97,3 +97,66 @@ def test_readiness_fails_closed_when_object_storage_unavailable(env): store.put = unavailable assert client.get("/ready").status_code == 503 assert client.get("/health").status_code == 200 + + +def test_slow_upload_fsync_does_not_block_worker_health(env, monkeypatch): + from concurrent.futures import ThreadPoolExecutor + from threading import Event + import os + + client, _, _, _, _ = env + auth, _, base = setup(client) + info = client.post(base + "/uploads", headers=auth, + json={"content_hash": hashlib.sha256(b"abc").hexdigest(), "size": 3}).json() + path = base + "/uploads/" + info["upload_id"] + entered, release = Event(), Event() + original = os.fsync + + def delayed(fd): + entered.set() + if not release.wait(10): + raise TimeoutError("test did not release fsync") + original(fd) + + monkeypatch.setattr(os, "fsync", delayed) + with ThreadPoolExecutor(max_workers=2) as pool: + pending = pool.submit(client.put, path + "?offset=0", headers=auth, content=b"abc") + try: + assert entered.wait(5) + # Both requests use this TestClient's single ASGI event loop. + health = pool.submit(client.get, "/health").result(timeout=2) + assert health.status_code == 200 + assert not pending.done() + finally: + release.set() + assert pending.result(timeout=5).json() == {"offset": 3} + assert client.get(path, headers=auth).json()["offset"] == 3 + assert client.post(path + "/complete", headers=auth).status_code == 200 + + +def test_upload_limits_and_failed_fsync_preserve_durable_offset(env, monkeypatch): + import os + + client, _, _, staging, _ = env + auth, _, base = setup(client) + data = b"a" * 1048576 + info = client.post(base + "/uploads", headers=auth, + json={"content_hash": hashlib.sha256(data).hexdigest(), "size": len(data)}).json() + path = base + "/uploads/" + info["upload_id"] + rejected = client.put(path + "?offset=0", headers=auth, content=data + b"x") + assert rejected.status_code == 413 + assert rejected.json()["error"]["code"] == "CHUNK_TOO_LARGE" + assert (staging / info["upload_id"]).stat().st_size == 0 + + def failed(_fd): + raise OSError("controlled fsync failure") + + with monkeypatch.context() as patch: + patch.setattr(os, "fsync", failed) + with pytest.raises(OSError, match="controlled fsync failure"): + client.put(path + "?offset=0", headers=auth, content=data) + # Failure propagated from the thread; the SQL transaction did not advance. + assert client.get(path, headers=auth).json()["offset"] == 0 + assert (staging / info["upload_id"]).stat().st_size == 0 + assert client.put(path + "?offset=0", headers=auth, content=data).json() == {"offset": len(data)} + assert client.post(path + "/complete", headers=auth).status_code == 200