fix(sync): 将上传持久化移出 ASGI 事件循环

This commit is contained in:
2026-09-09 06:36:42 +08:00
parent cfed432b7d
commit bab8d5ee4e
3 changed files with 80 additions and 2 deletions
@@ -644,3 +644,11 @@ Core 的独立数据目录目前不等于已授权 Vault。Python 旧笔记写
- 对照(生成式 100 MiB 流)、真实文件写入/恢复、重命名/删除、三种冲突解决、发现/重绑定各运行 3 次,全部通过。四个实际文件工作负载三次最大工作集分别约 12.70、12.63、12.72、12.91 MiB;最大提交内存分别约 2.75、2.85、2.82、2.92 MiB。没有改造前实测基线,不报告下降百分比。 - 对照(生成式 100 MiB 流)、真实文件写入/恢复、重命名/删除、三种冲突解决、发现/重绑定各运行 3 次,全部通过。四个实际文件工作负载三次最大工作集分别约 12.70、12.63、12.72、12.91 MiB;最大提交内存分别约 2.75、2.85、2.82、2.92 MiB。没有改造前实测基线,不报告下降百分比。
- 可审阅报告 [Windows Sync 组件内存观测](benchmarks/sync-memory-windows-5d3c214.md) 及 [原始 JSON](benchmarks/sync-memory-windows-5d3c214.json)。原始逐次测试日志留在 `.build/sync-memory-5d3c214/`。脚本 py_compile 通过,测量器和实际工作负载均已运行;本轮未改变 Rust 生产代码,未重复无关全量测试。 - 可审阅报告 [Windows Sync 组件内存观测](benchmarks/sync-memory-windows-5d3c214.md) 及 [原始 JSON](benchmarks/sync-memory-windows-5d3c214.json)。原始逐次测试日志留在 `.build/sync-memory-5d3c214/`。脚本 py_compile 通过,测量器和实际工作负载均已运行;本轮未改变 Rust 生产代码,未重复无关全量测试。
- scope 仅为单个 Rust 组件测试进程,不包括 WebView/Core/服务端/系统文件缓存;没有测量规划基准的磁盘/网络/10000 笔记/双账号双 Vault,也不是 S-09 四并发上传服务总 RSS 的证明。报告明确 NOT_ASSESSED,完整基准、部署和生产化目标保持未完成。 - scope 仅为单个 Rust 组件测试进程,不包括 WebView/Core/服务端/系统文件缓存;没有测量规划基准的磁盘/网络/10000 笔记/双账号双 Vault,也不是 S-09 四并发上传服务总 RSS 的证明。报告明确 NOT_ASSESSED,完整基准、部署和生产化目标保持未完成。
## 增量:Sync 分块上传不阻塞 ASGI 事件循环
- 复核服务端现状:complete 已用 hashlib.file_digest 与 put_filedownload 已逐块校验到临时文件再 FileResponse;未重复改造已存在的流式路径。
- upload_chunk 原先在 async 路由中直接执行同步数据库锁等待、文件写入和 fsync,会阻塞该 worker 事件循环。现将整段事务/文件持久化交给 Starlette run_in_threadpool,连接在同一工作线程创建和提交,仍保持先刷盘再确认 offset 的协议顺序。接收块先检查剩余额度再扩展 bytearray,避免把超限块复制进应用缓冲区;ASGI 自身收到的块不计入该应用缓冲上限证明。
- 新增故障测试以 Event 持续阻塞 fsync,在同一 TestClient ASGI 事件循环中要求 /health 在 2 秒内响应且上传仍在等待;释放后检查偏移与 complete。另测 1 MiB 加 1 字节拒绝、fsync 异常事务回滚、未确认尾部协调截断、恰好 1 MiB 重试成功。
- uv run --project "server sync" pytest "server sync/tests" -q 全套 25 通过,耗时 7.80 秒,JUnit.build/sync-server-upload-threadpool.xml。依赖报告 2 项 TestClient/AnyIO 弃用警告,无测试失败。环境为 SQLite/DiskObjects 受控测试,不是 PostgreSQL/MinIO 两 worker 或 S-09 四并发 100 MiB 服务总 RSS/30 分钟负载证明;线程池饱和与真实部署验收仍需继续。完整生产化目标未完成。
+9 -2
View File
@@ -15,6 +15,7 @@ from fastapi import FastAPI, Header, Query, Request
from fastapi.exceptions import RequestValidationError from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, FileResponse from fastapi.responses import JSONResponse, FileResponse
from starlette.background import BackgroundTask from starlette.background import BackgroundTask
from starlette.concurrency import run_in_threadpool
from .database import Database, password_hash, row, rows, run from .database import Database, password_hash, row, rows, run
from .models import Commit, Login, Refresh, Upload, VaultCreate from .models import Commit, Login, Refresh, Upload, VaultCreate
@@ -257,9 +258,15 @@ def create_app(db: Database, objects, staging: Path, *, quota=1024**3, clock=tim
offset: int = Query(ge=0), authorization: str = Header(default="")): offset: int = Query(ge=0), authorization: str = Header(default="")):
data = bytearray() data = bytearray()
async for chunk in request.stream(): async for chunk in request.stream():
data.extend(chunk) if len(chunk) > 1048576 - len(data):
if len(data) > 1048576:
raise SyncError(413, "CHUNK_TOO_LARGE") raise SyncError(413, "CHUNK_TOO_LARGE")
data.extend(chunk)
# Keep the transaction and durable write on one worker thread. A slow
# database lock or fsync must not block this worker's ASGI event loop.
return await run_in_threadpool(persist_upload_chunk, vault_id, upload_id,
authorization, offset, data)
def persist_upload_chunk(vault_id, upload_id, authorization, offset, data):
with db.transaction() as conn: with db.transaction() as conn:
upload = authorized_upload(conn, vault_id, upload_id, authorization) upload = authorized_upload(conn, vault_id, upload_id, authorization)
reconcile_staging(upload) reconcile_staging(upload)
@@ -97,3 +97,66 @@ def test_readiness_fails_closed_when_object_storage_unavailable(env):
store.put = unavailable store.put = unavailable
assert client.get("/ready").status_code == 503 assert client.get("/ready").status_code == 503
assert client.get("/health").status_code == 200 assert client.get("/health").status_code == 200
def test_slow_upload_fsync_does_not_block_worker_health(env, monkeypatch):
from concurrent.futures import ThreadPoolExecutor
from threading import Event
import os
client, _, _, _, _ = env
auth, _, base = setup(client)
info = client.post(base + "/uploads", headers=auth,
json={"content_hash": hashlib.sha256(b"abc").hexdigest(), "size": 3}).json()
path = base + "/uploads/" + info["upload_id"]
entered, release = Event(), Event()
original = os.fsync
def delayed(fd):
entered.set()
if not release.wait(10):
raise TimeoutError("test did not release fsync")
original(fd)
monkeypatch.setattr(os, "fsync", delayed)
with ThreadPoolExecutor(max_workers=2) as pool:
pending = pool.submit(client.put, path + "?offset=0", headers=auth, content=b"abc")
try:
assert entered.wait(5)
# Both requests use this TestClient's single ASGI event loop.
health = pool.submit(client.get, "/health").result(timeout=2)
assert health.status_code == 200
assert not pending.done()
finally:
release.set()
assert pending.result(timeout=5).json() == {"offset": 3}
assert client.get(path, headers=auth).json()["offset"] == 3
assert client.post(path + "/complete", headers=auth).status_code == 200
def test_upload_limits_and_failed_fsync_preserve_durable_offset(env, monkeypatch):
import os
client, _, _, staging, _ = env
auth, _, base = setup(client)
data = b"a" * 1048576
info = client.post(base + "/uploads", headers=auth,
json={"content_hash": hashlib.sha256(data).hexdigest(), "size": len(data)}).json()
path = base + "/uploads/" + info["upload_id"]
rejected = client.put(path + "?offset=0", headers=auth, content=data + b"x")
assert rejected.status_code == 413
assert rejected.json()["error"]["code"] == "CHUNK_TOO_LARGE"
assert (staging / info["upload_id"]).stat().st_size == 0
def failed(_fd):
raise OSError("controlled fsync failure")
with monkeypatch.context() as patch:
patch.setattr(os, "fsync", failed)
with pytest.raises(OSError, match="controlled fsync failure"):
client.put(path + "?offset=0", headers=auth, content=data)
# Failure propagated from the thread; the SQL transaction did not advance.
assert client.get(path, headers=auth).json()["offset"] == 0
assert (staging / info["upload_id"]).stat().st_size == 0
assert client.put(path + "?offset=0", headers=auth, content=data).json() == {"offset": len(data)}
assert client.post(path + "/complete", headers=auth).status_code == 200