fix(backend): 落实 PR #12 评审意见

- P1 事件循环让出:run_rag 在样本边界 await asyncio.sleep(0),运行中取消/进度/SSE 可及时调度
- P2 SSE 终止事件:历史回放期间识别终止事件并结束流,try/finally 保证订阅清理
- P2 FTS 截断:fts 走数据库侧精确分页与计数,阈值经 bm25 截止值换算,不再受 5000 条固定截断
- P2 仅块标注:expected_block_ids 从块反查所属笔记,避免合法样本被判零分

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
yxx
2026-09-03 23:37:09 +08:00
co-authored by Claude Code
parent 3898530585
commit 3bd475dc15
6 changed files with 331 additions and 48 deletions
+45 -17
View File
@@ -29,8 +29,6 @@ from app.textutils import make_snippet, match_query
CANDIDATE_POOL = 50
# 分页窗口上限:候选池至少覆盖 offset+limit,但设上限防止超大 offset 撑爆内存
MAX_CANDIDATE_POOL = 200
# FTS 全量取回上限:统一归一化 + 阈值过滤后再分页,保证阈值语义跨页一致
FTS_FETCH_LIMIT = 5000
# 带 metadata 过滤时放大召回倍数,缓解「先截断候选池再过滤」造成的漏召回
OVERSCAN_FACTOR = 4
@@ -58,7 +56,7 @@ class RetrievalEngine:
# 候选池至少覆盖本次请求的 offset+limit,保证分页能取到目标页;设上限防内存失控
window = min(request.offset + request.limit, MAX_CANDIDATE_POOL)
pool_size = max(CANDIDATE_POOL, window)
# 带过滤时放大召回;FTS 则一次性取全量命中(≤FTS_FETCH_LIMIT)避免截断漏召回
# 带过滤时放大召回,缓解「先截断候选池再过滤」造成的漏召回
recall = min(pool_size * OVERSCAN_FACTOR, MAX_CANDIDATE_POOL) if has_filters else pool_size
# 1. 按模式收集候选(FTS 与 Vector 各产出「按相关性降序」的 block_id 列表)
@@ -126,7 +124,7 @@ class RetrievalEngine:
# score_threshold:归一化后过滤低分结果(默认 0 不过滤)
ordered = [(bid, score) for bid, score in ordered if score >= request.score_threshold]
# 5. 分页:total = 过滤后候选集大小。fts 已取全量(≤FTS_FETCH_LIMIT)故为真实命中数;
# 5. 分页:total = 过滤后候选集大小。fts 走数据库精确分页,total 为真实命中数;
# vector/hybrid 为 KNN 候选集,无全局 total。
total = len(ordered)
page = ordered[request.offset : request.offset + request.limit]
@@ -139,18 +137,18 @@ class RetrievalEngine:
)
def _search_fts(self, request: SearchRequest) -> SearchResponse:
"""FTS 专用路径:先取全量命中(≤FTS_FETCH_LIMIT),统一归一化 + 阈值过滤后再分页
"""FTS 专用路径:在数据库侧完成过滤、计数与分页,不取全量后再截断
阈值过滤必须在计数与分页之前完成,否则 score_threshold 只作用于当前页,
且返回的 total 与 items 数量不一致(如 items 为空但 total 非零)。"""
阈值过滤时,min-max 归一化是 bm25 的线性函数,据此把 score_threshold 换算为
bm25 截止值(bm25_max),使过滤、计数与分页口径一致;无阈值时走数据库原生分页,
total 始终为过滤后的真实命中数,不再受固定截断影响。
"""
match = match_query(request.query)
if not match:
return self._empty(request)
fts_hits, _ = repository.fts_search_page(
bounds = repository.fts_score_bounds(
match=match,
limit=FTS_FETCH_LIMIT,
offset=0,
folders=request.folders,
note_ids=request.note_ids,
tags=request.tags,
@@ -159,17 +157,47 @@ class RetrievalEngine:
updated_from=request.updated_from,
updated_to=request.updated_to,
)
if not fts_hits:
if bounds is None:
return self._empty(request)
ordered = normalize_scores([(hit.block_id, -hit.bm25) for hit in fts_hits])
ordered = [(bid, score) for bid, score in ordered if score >= request.score_threshold]
total = len(ordered)
page = ordered[request.offset : request.offset + request.limit]
hits = {h.block_id: h for h in repository.get_block_hits([bid for bid, _ in page])}
lo, hi = bounds
bm25_max: float | None = None
if request.score_threshold > 0:
# norm = (hi - bm25) / (hi - lo)norm >= threshold ⟺ bm25 <= hi - threshold*(hi - lo)
bm25_max = hi - request.score_threshold * (hi - lo)
fts_hits, total = repository.fts_search_page(
match=match,
limit=request.limit,
offset=request.offset,
folders=request.folders,
note_ids=request.note_ids,
tags=request.tags,
created_from=request.created_from,
created_to=request.created_to,
updated_from=request.updated_from,
updated_to=request.updated_to,
bm25_max=bm25_max,
)
if not fts_hits:
# 本页无结果:offset 越过末页时 total 仍为真实命中数(>0),需保留而非归零
return SearchResponse(
query=request.query,
mode=request.mode,
items=[],
page=PageMeta(total=total, limit=request.limit, offset=request.offset),
)
# 分数按全局 bm25 上下界归一化(与取全量后 normalize_scores 等价),保证跨页一致
span = hi - lo
if span == 0:
ordered = [(hit.block_id, 1.0) for hit in fts_hits]
else:
ordered = [(hit.block_id, round((hi - hit.bm25) / span, 6)) for hit in fts_hits]
hits = {h.block_id: h for h in repository.get_block_hits([bid for bid, _ in ordered])}
items = [
self._build_result(hits[block_id], request, score)
for block_id, score in page
for block_id, score in ordered
if block_id in hits
]
return SearchResponse(