fix(backend): 落实 PR #12 评审意见
- P1 事件循环让出:run_rag 在样本边界 await asyncio.sleep(0),运行中取消/进度/SSE 可及时调度 - P2 SSE 终止事件:历史回放期间识别终止事件并结束流,try/finally 保证订阅清理 - P2 FTS 截断:fts 走数据库侧精确分页与计数,阈值经 bm25 截止值换算,不再受 5000 条固定截断 - P2 仅块标注:expected_block_ids 从块反查所属笔记,避免合法样本被判零分 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -29,8 +29,6 @@ from app.textutils import make_snippet, match_query
|
||||
CANDIDATE_POOL = 50
|
||||
# 分页窗口上限:候选池至少覆盖 offset+limit,但设上限防止超大 offset 撑爆内存
|
||||
MAX_CANDIDATE_POOL = 200
|
||||
# FTS 全量取回上限:统一归一化 + 阈值过滤后再分页,保证阈值语义跨页一致
|
||||
FTS_FETCH_LIMIT = 5000
|
||||
# 带 metadata 过滤时放大召回倍数,缓解「先截断候选池再过滤」造成的漏召回
|
||||
OVERSCAN_FACTOR = 4
|
||||
|
||||
@@ -58,7 +56,7 @@ class RetrievalEngine:
|
||||
# 候选池至少覆盖本次请求的 offset+limit,保证分页能取到目标页;设上限防内存失控
|
||||
window = min(request.offset + request.limit, MAX_CANDIDATE_POOL)
|
||||
pool_size = max(CANDIDATE_POOL, window)
|
||||
# 带过滤时放大召回;FTS 则一次性取全量命中(≤FTS_FETCH_LIMIT)避免截断漏召回
|
||||
# 带过滤时放大召回,缓解「先截断候选池再过滤」造成的漏召回
|
||||
recall = min(pool_size * OVERSCAN_FACTOR, MAX_CANDIDATE_POOL) if has_filters else pool_size
|
||||
|
||||
# 1. 按模式收集候选(FTS 与 Vector 各产出「按相关性降序」的 block_id 列表)
|
||||
@@ -126,7 +124,7 @@ class RetrievalEngine:
|
||||
# score_threshold:归一化后过滤低分结果(默认 0 不过滤)
|
||||
ordered = [(bid, score) for bid, score in ordered if score >= request.score_threshold]
|
||||
|
||||
# 5. 分页:total = 过滤后候选集大小。fts 已取全量(≤FTS_FETCH_LIMIT)故为真实命中数;
|
||||
# 5. 分页:total = 过滤后候选集大小。fts 走数据库精确分页,total 为真实命中数;
|
||||
# vector/hybrid 为 KNN 候选集,无全局 total。
|
||||
total = len(ordered)
|
||||
page = ordered[request.offset : request.offset + request.limit]
|
||||
@@ -139,18 +137,18 @@ class RetrievalEngine:
|
||||
)
|
||||
|
||||
def _search_fts(self, request: SearchRequest) -> SearchResponse:
|
||||
"""FTS 专用路径:先取全量命中(≤FTS_FETCH_LIMIT),统一归一化 + 阈值过滤后再分页。
|
||||
"""FTS 专用路径:在数据库侧完成过滤、计数与分页,不取全量后再截断。
|
||||
|
||||
阈值过滤必须在计数与分页之前完成,否则 score_threshold 只作用于当前页,
|
||||
且返回的 total 与 items 数量不一致(如 items 为空但 total 非零)。"""
|
||||
阈值过滤时,min-max 归一化是 bm25 的线性函数,据此把 score_threshold 换算为
|
||||
bm25 截止值(bm25_max),使过滤、计数与分页口径一致;无阈值时走数据库原生分页,
|
||||
total 始终为过滤后的真实命中数,不再受固定截断影响。
|
||||
"""
|
||||
match = match_query(request.query)
|
||||
if not match:
|
||||
return self._empty(request)
|
||||
|
||||
fts_hits, _ = repository.fts_search_page(
|
||||
bounds = repository.fts_score_bounds(
|
||||
match=match,
|
||||
limit=FTS_FETCH_LIMIT,
|
||||
offset=0,
|
||||
folders=request.folders,
|
||||
note_ids=request.note_ids,
|
||||
tags=request.tags,
|
||||
@@ -159,17 +157,47 @@ class RetrievalEngine:
|
||||
updated_from=request.updated_from,
|
||||
updated_to=request.updated_to,
|
||||
)
|
||||
if not fts_hits:
|
||||
if bounds is None:
|
||||
return self._empty(request)
|
||||
|
||||
ordered = normalize_scores([(hit.block_id, -hit.bm25) for hit in fts_hits])
|
||||
ordered = [(bid, score) for bid, score in ordered if score >= request.score_threshold]
|
||||
total = len(ordered)
|
||||
page = ordered[request.offset : request.offset + request.limit]
|
||||
hits = {h.block_id: h for h in repository.get_block_hits([bid for bid, _ in page])}
|
||||
lo, hi = bounds
|
||||
bm25_max: float | None = None
|
||||
if request.score_threshold > 0:
|
||||
# norm = (hi - bm25) / (hi - lo);norm >= threshold ⟺ bm25 <= hi - threshold*(hi - lo)
|
||||
bm25_max = hi - request.score_threshold * (hi - lo)
|
||||
|
||||
fts_hits, total = repository.fts_search_page(
|
||||
match=match,
|
||||
limit=request.limit,
|
||||
offset=request.offset,
|
||||
folders=request.folders,
|
||||
note_ids=request.note_ids,
|
||||
tags=request.tags,
|
||||
created_from=request.created_from,
|
||||
created_to=request.created_to,
|
||||
updated_from=request.updated_from,
|
||||
updated_to=request.updated_to,
|
||||
bm25_max=bm25_max,
|
||||
)
|
||||
if not fts_hits:
|
||||
# 本页无结果:offset 越过末页时 total 仍为真实命中数(>0),需保留而非归零
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
mode=request.mode,
|
||||
items=[],
|
||||
page=PageMeta(total=total, limit=request.limit, offset=request.offset),
|
||||
)
|
||||
|
||||
# 分数按全局 bm25 上下界归一化(与取全量后 normalize_scores 等价),保证跨页一致
|
||||
span = hi - lo
|
||||
if span == 0:
|
||||
ordered = [(hit.block_id, 1.0) for hit in fts_hits]
|
||||
else:
|
||||
ordered = [(hit.block_id, round((hi - hit.bm25) / span, 6)) for hit in fts_hits]
|
||||
hits = {h.block_id: h for h in repository.get_block_hits([bid for bid, _ in ordered])}
|
||||
items = [
|
||||
self._build_result(hits[block_id], request, score)
|
||||
for block_id, score in page
|
||||
for block_id, score in ordered
|
||||
if block_id in hits
|
||||
]
|
||||
return SearchResponse(
|
||||
|
||||
Reference in New Issue
Block a user