fix(retrieval): 修复路径逃逸、部分提交、失效向量与分页问题
- 路径逃逸:清洗 folder(拒绝 ..、绝对路径/盘符),_abs_path 增加 Vault 边界校验 - 部分提交:create/update 索引失败时回滚文件 - 失效向量残留:replace_note_metadata 返回旧 id,index_note 清理旧向量 - 分页不完整:fts_count 返回真实命中总数,候选池覆盖 offset+limit Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -27,6 +27,8 @@ from app.textutils import make_snippet, match_query
|
||||
|
||||
# 每个通道的候选池大小;真实规模上来后按 Retrieval Config 调整
|
||||
CANDIDATE_POOL = 50
|
||||
# 分页窗口上限:候选池至少覆盖 offset+limit,但设上限防止超大 offset 撑爆内存
|
||||
MAX_CANDIDATE_POOL = 200
|
||||
|
||||
|
||||
class RetrievalEngine:
|
||||
@@ -41,23 +43,30 @@ class RetrievalEngine:
|
||||
self.vector_store = vector_store
|
||||
|
||||
async def search(self, request: SearchRequest) -> SearchResponse:
|
||||
# 候选池至少覆盖本次请求的 offset+limit,保证分页能取到目标页;设上限防内存失控
|
||||
window = min(request.offset + request.limit, MAX_CANDIDATE_POOL)
|
||||
pool_size = max(CANDIDATE_POOL, window)
|
||||
|
||||
# 1. 按模式收集候选(FTS 与 Vector 各产出「按相关性降序」的 block_id 列表)
|
||||
fts_ranked: list[str] = []
|
||||
vec_ranked: list[str] = []
|
||||
fts_scores: dict[str, float] = {}
|
||||
vec_scores: dict[str, float] = {}
|
||||
fts_total = 0
|
||||
|
||||
if request.mode in (SearchMode.fts, SearchMode.hybrid):
|
||||
match = match_query(request.query)
|
||||
if match:
|
||||
fts_hits = repository.fts_search(match, CANDIDATE_POOL)
|
||||
fts_hits = repository.fts_search(match, pool_size)
|
||||
fts_ranked = [h.block_id for h in fts_hits]
|
||||
# bm25 越小越相关,取反后统一为「越大越相关」
|
||||
fts_scores = {h.block_id: -h.bm25 for h in fts_hits}
|
||||
if request.mode == SearchMode.fts:
|
||||
fts_total = repository.fts_count(match)
|
||||
|
||||
if request.mode in (SearchMode.vector, SearchMode.hybrid):
|
||||
query_vec = await self.embedding.embed_query(request.query)
|
||||
vec_hits = await self.vector_store.search(query_vec, top_k=CANDIDATE_POOL)
|
||||
vec_hits = await self.vector_store.search(query_vec, top_k=pool_size)
|
||||
vec_ranked = [v.id for v in vec_hits]
|
||||
vec_scores = {v.id: v.score for v in vec_hits}
|
||||
|
||||
@@ -95,8 +104,11 @@ class RetrievalEngine:
|
||||
|
||||
ordered = normalize_scores(ordered)
|
||||
|
||||
# 5. 分页
|
||||
total = len(ordered)
|
||||
# 5. 分页:fts 用真实命中总数;vector/hybrid 为 KNN 候选集,无全局 total
|
||||
if request.mode == SearchMode.fts:
|
||||
total = fts_total
|
||||
else:
|
||||
total = len(ordered)
|
||||
page = ordered[request.offset : request.offset + request.limit]
|
||||
items = [self._build_result(hits[block_id], request, score) for block_id, score in page]
|
||||
return SearchResponse(
|
||||
|
||||
Reference in New Issue
Block a user