Perf/frontend chunk loading优化长文渲染、后台运行与向量检索,补齐运行日志和并发一致性 #34
@@ -36,6 +36,7 @@ PR #31 已合并。工作区打开与 HTTP 保存不再等待向量推理;正
|
|||||||
新增开发说明:
|
新增开发说明:
|
||||||
|
|
||||||
- [工作区后台索引与保存](docs/development/工作区后台索引与保存开发说明.md):状态、并发、恢复和验证。
|
- [工作区后台索引与保存](docs/development/工作区后台索引与保存开发说明.md):状态、并发、恢复和验证。
|
||||||
|
- [模型隔离向量索引与增量登记](docs/development/模型隔离向量索引与增量登记.md):持久化 sqlite-vec 空间、旧向量复用、外部新增文件增量计算与检索性能验证。
|
||||||
- [Mermaid 预览与缩放](docs/development/Mermaid预览与缩放开发说明.md):大图适配、鼠标缩放和文字裁切修复。
|
- [Mermaid 预览与缩放](docs/development/Mermaid预览与缩放开发说明.md):大图适配、鼠标缩放和文字裁切修复。
|
||||||
- [扩展安装持久化与社区包](docs/development/扩展安装持久化与社区包开发说明.md):安装边界和示例包验证。
|
- [扩展安装持久化与社区包](docs/development/扩展安装持久化与社区包开发说明.md):安装边界和示例包验证。
|
||||||
- [模型上下文管理](docs/development/模型上下文管理.md):全局人设、预算估算和摘要限制。
|
- [模型上下文管理](docs/development/模型上下文管理.md):全局人设、预算估算和摘要限制。
|
||||||
|
|||||||
@@ -2,15 +2,14 @@
|
|||||||
|
|
||||||
The runtime's model_id is the authoritative space ID (including provider URL,
|
The runtime's model_id is the authoritative space ID (including provider URL,
|
||||||
endpoint, model and dimensions); equal dimensions alone never imply compatibility.
|
endpoint, model and dimensions); equal dimensions alone never imply compatibility.
|
||||||
This phase uses a lazy, rebuildable SQLite side table instead of a schema migration.
|
Durable vectors are reused to build per-space/dimension sqlite-vec indexes lazily.
|
||||||
Search scans only current blocks in one database snapshot and requires complete
|
Native exact KNN avoids Python JSON decoding and dot products on every search.
|
||||||
coverage. Cosine ranking costs O(blocks * dimensions) with an O(top_k) heap; this
|
Coverage checks and ranking share one transaction.
|
||||||
small-vault implementation should become a per-space ANN index at larger scale.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import heapq
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import math
|
import math
|
||||||
@@ -24,6 +23,7 @@ from app.operation_logs import log_event
|
|||||||
from app.retrieval.vectorstore import VectorHit
|
from app.retrieval.vectorstore import VectorHit
|
||||||
from app.retrieval.provenance import record_embedding
|
from app.retrieval.provenance import record_embedding
|
||||||
from app.retrieval.hybrid import rrf_fuse
|
from app.retrieval.hybrid import rrf_fuse
|
||||||
|
from app.retrieval import space_index
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -121,9 +121,15 @@ def _ensure_table(conn: sqlite3.Connection) -> None:
|
|||||||
block_id TEXT NOT NULL REFERENCES blocks(block_id) ON DELETE CASCADE,
|
block_id TEXT NOT NULL REFERENCES blocks(block_id) ON DELETE CASCADE,
|
||||||
dimensions INTEGER NOT NULL CHECK (dimensions > 0),
|
dimensions INTEGER NOT NULL CHECK (dimensions > 0),
|
||||||
vector TEXT NOT NULL,
|
vector TEXT NOT NULL,
|
||||||
PRIMARY KEY (space_id, block_id)
|
PRIMARY KEY (space_id, dimensions, block_id)
|
||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
|
primary = [row[1] for row in sorted(conn.execute('PRAGMA table_info(routed_block_vectors)'), key=lambda row: row[5]) if row[5]]
|
||||||
|
if primary == ['space_id', 'block_id']:
|
||||||
|
conn.execute('CREATE TABLE routed_block_vectors_upgrade (space_id TEXT NOT NULL, block_id TEXT NOT NULL REFERENCES blocks(block_id) ON DELETE CASCADE, dimensions INTEGER NOT NULL CHECK(dimensions>0), vector TEXT NOT NULL, PRIMARY KEY(space_id,dimensions,block_id))')
|
||||||
|
conn.execute('INSERT INTO routed_block_vectors_upgrade SELECT * FROM routed_block_vectors')
|
||||||
|
conn.execute('DROP TABLE routed_block_vectors')
|
||||||
|
conn.execute('ALTER TABLE routed_block_vectors_upgrade RENAME TO routed_block_vectors')
|
||||||
conn.execute("""
|
conn.execute("""
|
||||||
CREATE INDEX IF NOT EXISTS routed_block_vectors_block_id
|
CREATE INDEX IF NOT EXISTS routed_block_vectors_block_id
|
||||||
ON routed_block_vectors(block_id)
|
ON routed_block_vectors(block_id)
|
||||||
@@ -149,13 +155,14 @@ def store_remote(
|
|||||||
conn.executemany(
|
conn.executemany(
|
||||||
"""INSERT INTO routed_block_vectors (space_id, block_id, dimensions, vector)
|
"""INSERT INTO routed_block_vectors (space_id, block_id, dimensions, vector)
|
||||||
VALUES (?, ?, ?, ?)
|
VALUES (?, ?, ?, ?)
|
||||||
ON CONFLICT (space_id, block_id) DO UPDATE SET
|
ON CONFLICT (space_id, dimensions, block_id) DO UPDATE SET
|
||||||
dimensions = excluded.dimensions, vector = excluded.vector""",
|
dimensions = excluded.dimensions, vector = excluded.vector""",
|
||||||
[
|
[
|
||||||
(batch.space_id, block_id, batch.dimensions, json.dumps(vector, allow_nan=False))
|
(batch.space_id, block_id, batch.dimensions, json.dumps(vector, allow_nan=False))
|
||||||
for block_id, vector in zip(block_ids, batch.vectors)
|
for block_id, vector in zip(block_ids, batch.vectors)
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
space_index.upsert(conn, block_ids, batch)
|
||||||
except BaseException:
|
except BaseException:
|
||||||
conn.execute("ROLLBACK TO routed_vectors_write")
|
conn.execute("ROLLBACK TO routed_vectors_write")
|
||||||
raise
|
raise
|
||||||
@@ -183,6 +190,10 @@ async def search_remote(query: str, *, top_k: int, accept_local=False, strict=Fa
|
|||||||
if batch is None:
|
if batch is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
return await asyncio.to_thread(_search_space, batch, top_k, strict)
|
||||||
|
|
||||||
|
|
||||||
|
def _search_space(batch, top_k, strict):
|
||||||
record_embedding(attempted_space={"model_id": batch.space_id, "dimensions": batch.dimensions})
|
record_embedding(attempted_space={"model_id": batch.space_id, "dimensions": batch.dimensions})
|
||||||
try:
|
try:
|
||||||
conn = connect()
|
conn = connect()
|
||||||
@@ -198,29 +209,8 @@ async def search_remote(query: str, *, top_k: int, accept_local=False, strict=Fa
|
|||||||
if strict:
|
if strict:
|
||||||
raise ValueError("semantic index missing")
|
raise ValueError("semantic index missing")
|
||||||
return None
|
return None
|
||||||
rows = conn.execute(
|
_ensure_table(conn)
|
||||||
"""SELECT b.block_id, r.vector
|
result = space_index.search(conn, batch, top_k)
|
||||||
FROM blocks AS b
|
|
||||||
LEFT JOIN routed_block_vectors AS r
|
|
||||||
ON r.block_id = b.block_id AND r.space_id = ? AND r.dimensions = ?
|
|
||||||
ORDER BY b.block_id""",
|
|
||||||
(batch.space_id, batch.dimensions),
|
|
||||||
)
|
|
||||||
|
|
||||||
def hits():
|
|
||||||
for row in rows:
|
|
||||||
if row["vector"] is None:
|
|
||||||
raise ValueError("remote space has incomplete block coverage")
|
|
||||||
vector = _unit_vector(json.loads(row["vector"]), batch.dimensions)
|
|
||||||
score = math.fsum(a * b for a, b in zip(batch.vectors[0], vector))
|
|
||||||
yield VectorHit(id=row["block_id"], score=max(0.0, min(1.0, score)))
|
|
||||||
|
|
||||||
try:
|
|
||||||
result = heapq.nlargest(top_k, hits(), key=lambda hit: hit.score)
|
|
||||||
finally:
|
|
||||||
# Exceptions may retain the generator/traceback; finalize its
|
|
||||||
# cursor now so a subsequent rebuild can acquire a write lock.
|
|
||||||
rows.close()
|
|
||||||
record_embedding(source=batch.source, model_id=batch.space_id,
|
record_embedding(source=batch.source, model_id=batch.space_id,
|
||||||
dimensions=batch.dimensions, fallback_reason=None)
|
dimensions=batch.dimensions, fallback_reason=None)
|
||||||
return result
|
return result
|
||||||
@@ -244,6 +234,10 @@ async def _search_partitioned(query: str, policies: set[bool], *, top_k: int, st
|
|||||||
if batch is None:
|
if batch is None:
|
||||||
return None
|
return None
|
||||||
batches[policy] = batch
|
batches[policy] = batch
|
||||||
|
return await asyncio.to_thread(_search_partitions, batches, policies, top_k, strict)
|
||||||
|
|
||||||
|
|
||||||
|
def _search_partitions(batches, policies, top_k, strict):
|
||||||
conn = connect()
|
conn = connect()
|
||||||
try:
|
try:
|
||||||
with transaction(conn):
|
with transaction(conn):
|
||||||
@@ -253,23 +247,8 @@ async def _search_partitioned(query: str, policies: set[bool], *, top_k: int, st
|
|||||||
raise ValueError("embedding policies changed while querying")
|
raise ValueError("embedding policies changed while querying")
|
||||||
ranked = []
|
ranked = []
|
||||||
for policy, batch in batches.items():
|
for policy, batch in batches.items():
|
||||||
rows = conn.execute(
|
_ensure_table(conn)
|
||||||
"SELECT b.block_id,r.vector FROM blocks b LEFT JOIN routed_block_vectors r "
|
ranked.append(space_index.search(conn, batch, top_k, policy))
|
||||||
"ON r.block_id=b.block_id AND r.space_id=? AND r.dimensions=? "
|
|
||||||
"WHERE b.embedding_local_only=? ORDER BY b.block_id",
|
|
||||||
(batch.space_id, batch.dimensions, int(policy)),
|
|
||||||
)
|
|
||||||
def hits():
|
|
||||||
for row in rows:
|
|
||||||
if row['vector'] is None:
|
|
||||||
raise ValueError("incomplete policy coverage")
|
|
||||||
vector = _unit_vector(json.loads(row['vector']), batch.dimensions)
|
|
||||||
score = math.fsum(a * b for a, b in zip(batch.vectors[0], vector))
|
|
||||||
yield VectorHit(id=row['block_id'], score=max(0.0, min(1.0, score)))
|
|
||||||
try:
|
|
||||||
ranked.append(heapq.nlargest(top_k, hits(), key=lambda hit: hit.score))
|
|
||||||
finally:
|
|
||||||
rows.close()
|
|
||||||
spaces = [{"source": b.source, "model_id": b.space_id, "dimensions": b.dimensions,
|
spaces = [{"source": b.source, "model_id": b.space_id, "dimensions": b.dimensions,
|
||||||
"local_only": policy} for policy, b in batches.items()]
|
"local_only": policy} for policy, b in batches.items()]
|
||||||
record_embedding(source="mixed" if len({b.source for b in batches.values()}) > 1 else batch.source,
|
record_embedding(source="mixed" if len({b.source for b in batches.values()}) > 1 else batch.source,
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
"""Persistent vec0 indexes derived from durable routed vectors, one per space/dimension."""
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
|
||||||
|
import sqlite_vec
|
||||||
|
|
||||||
|
from app.retrieval.vectorstore import VectorHit
|
||||||
|
|
||||||
|
|
||||||
|
def table_name(space, dimensions):
|
||||||
|
return 'routed_vec_' + hashlib.sha256(json.dumps([space, dimensions]).encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def ensure(conn, space, dimensions):
|
||||||
|
from app.retrieval.routed_vectors import _unit_vector
|
||||||
|
table = table_name(space, dimensions)
|
||||||
|
if conn.execute('SELECT 1 FROM sqlite_master WHERE name=?', (table,)).fetchone():
|
||||||
|
return table
|
||||||
|
if type(dimensions) is not int or not 0 < dimensions <= 8192:
|
||||||
|
raise ValueError('unsupported vector dimensions')
|
||||||
|
conn.execute(f'CREATE VIRTUAL TABLE {table} USING vec0(block_id TEXT PRIMARY KEY, embedding float[{dimensions}], local_only INTEGER)')
|
||||||
|
for row in conn.execute('SELECT r.block_id,r.vector,b.embedding_local_only FROM routed_block_vectors r JOIN blocks b USING(block_id) WHERE r.space_id=? AND r.dimensions=?', (space, dimensions)):
|
||||||
|
conn.execute(f'INSERT INTO {table}(block_id,embedding,local_only) VALUES (?,?,?)',
|
||||||
|
(row[0], sqlite_vec.serialize_float32(_unit_vector(json.loads(row[1]), dimensions)), row[2]))
|
||||||
|
literal = conn.execute('SELECT quote(?)', (space,)).fetchone()[0]
|
||||||
|
for event in ('DELETE', 'UPDATE'):
|
||||||
|
conn.execute(f'''CREATE TRIGGER {table}_{event.lower()} AFTER {event} ON routed_block_vectors
|
||||||
|
WHEN old.space_id={literal} AND old.dimensions={dimensions}
|
||||||
|
BEGIN DELETE FROM {table} WHERE block_id=old.block_id; END''')
|
||||||
|
return table
|
||||||
|
|
||||||
|
|
||||||
|
def upsert(conn, block_ids, batch):
|
||||||
|
from app.retrieval.routed_vectors import _unit_vector
|
||||||
|
table = ensure(conn, batch.space_id, batch.dimensions)
|
||||||
|
for block_id, vector in zip(block_ids, batch.vectors):
|
||||||
|
conn.execute(f'DELETE FROM {table} WHERE block_id=?', (block_id,))
|
||||||
|
conn.execute(f'INSERT INTO {table}(block_id,embedding,local_only) SELECT block_id,?,embedding_local_only FROM blocks WHERE block_id=?',
|
||||||
|
(sqlite_vec.serialize_float32(_unit_vector(vector, batch.dimensions)), block_id))
|
||||||
|
|
||||||
|
|
||||||
|
def search(conn, batch, top_k, policy=None):
|
||||||
|
table = ensure(conn, batch.space_id, batch.dimensions)
|
||||||
|
# Coverage checks stay relational; no JSON decoding or Python dot products on the hot path.
|
||||||
|
where = '' if policy is None else ' AND b.embedding_local_only=?'
|
||||||
|
params = () if policy is None else (int(policy),)
|
||||||
|
missing = conn.execute(f'''SELECT 1 FROM blocks b LEFT JOIN routed_block_vectors r
|
||||||
|
ON r.block_id=b.block_id AND r.space_id=? AND r.dimensions=?
|
||||||
|
WHERE r.block_id IS NULL{where} LIMIT 1''', (batch.space_id, batch.dimensions, *params)).fetchone()
|
||||||
|
expected = conn.execute('SELECT COUNT(*) FROM blocks' + ('' if policy is None else ' WHERE embedding_local_only=?'), params).fetchone()[0]
|
||||||
|
actual = conn.execute(f'SELECT COUNT(*) FROM {table}' + ('' if policy is None else ' WHERE local_only=?'), params).fetchone()[0]
|
||||||
|
if missing or actual != expected:
|
||||||
|
raise ValueError('incomplete vector space coverage')
|
||||||
|
if top_k <= 0:
|
||||||
|
return []
|
||||||
|
rows = conn.execute(f'SELECT block_id,distance FROM {table} WHERE embedding MATCH ? AND k=?'
|
||||||
|
+ ('' if policy is None else ' AND local_only=?'),
|
||||||
|
(sqlite_vec.serialize_float32(batch.vectors[0]), top_k, *params)).fetchall()
|
||||||
|
return [VectorHit(id=row[0], score=max(0.0, min(1.0, 1 - row[1] ** 2 / 2))) for row in rows]
|
||||||
@@ -271,6 +271,14 @@ async def _refresh_saved_note(note_id: str) -> None:
|
|||||||
conn = connect()
|
conn = connect()
|
||||||
try:
|
try:
|
||||||
with transaction(conn):
|
with transaction(conn):
|
||||||
|
existing_ids = {row[0] for row in conn.execute('SELECT block_id FROM blocks WHERE note_id=?', (note_id,))}
|
||||||
|
if existing_ids != {block.block_id for block in parsed.blocks}:
|
||||||
|
# An external editor changed a newly registered note while inference ran.
|
||||||
|
# Reconcile that note only; the snapshot check above protects newer saves.
|
||||||
|
parsed.title = parse_note(markdown=markdown, file_path=record.file_path,
|
||||||
|
folder=record.folder, tags=record.tags, created_at=record.created_at,
|
||||||
|
updated_at=record.updated_at, note_id=note_id).title
|
||||||
|
await index_note(parsed, prepared=prepared, conn=conn)
|
||||||
# Write only vectors: metadata and FTS already represent the saved revision.
|
# Write only vectors: metadata and FTS already represent the saved revision.
|
||||||
vectors, remote = prepared
|
vectors, remote = prepared
|
||||||
from app.retrieval.vectorstore import VectorRecord
|
from app.retrieval.vectorstore import VectorRecord
|
||||||
@@ -278,6 +286,14 @@ async def _refresh_saved_note(note_id: str) -> None:
|
|||||||
await vector_store.upsert([VectorRecord(id=b.block_id, vector=v)
|
await vector_store.upsert([VectorRecord(id=b.block_id, vector=v)
|
||||||
for b, v in zip(parsed.blocks, vectors)], conn=conn)
|
for b, v in zip(parsed.blocks, vectors)], conn=conn)
|
||||||
routed_vectors.store_remote(conn, [b.block_id for b in parsed.blocks], remote)
|
routed_vectors.store_remote(conn, [b.block_id for b in parsed.blocks], remote)
|
||||||
|
if isinstance(note_service.embedding, LocalEmbedding) and parsed.blocks:
|
||||||
|
from app.retrieval.space_index import table_name
|
||||||
|
if remote is None:
|
||||||
|
raise ApiError(503, 'EMBEDDING_UNAVAILABLE', '笔记已保存,向量计算未完成。')
|
||||||
|
table = table_name(remote.space_id, remote.dimensions)
|
||||||
|
missing = conn.execute(f'SELECT 1 FROM blocks b LEFT JOIN {table} v ON v.block_id=b.block_id WHERE b.note_id=? AND v.block_id IS NULL LIMIT 1', (note_id,)).fetchone()
|
||||||
|
if missing:
|
||||||
|
raise ApiError(500, 'SEMANTIC_INDEX_WRITE_FAILED', '向量写入未完成,保留待处理标记。')
|
||||||
repository.set_index_meta({key: '0'}, conn=conn)
|
repository.set_index_meta({key: '0'}, conn=conn)
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|||||||
@@ -161,7 +161,7 @@ async def _register_workspace_files() -> None:
|
|||||||
created_at=parsed.created_at, updated_at=parsed.updated_at, blocks=parsed.blocks)
|
created_at=parsed.created_at, updated_at=parsed.updated_at, blocks=parsed.blocks)
|
||||||
conn.execute('UPDATE blocks SET embedding_local_only=? WHERE note_id=?', (int(parsed.embedding_local_only), parsed.note_id))
|
conn.execute('UPDATE blocks SET embedding_local_only=? WHERE note_id=?', (int(parsed.embedding_local_only), parsed.note_id))
|
||||||
if prepared:
|
if prepared:
|
||||||
repository.set_index_meta({'workspace_vectors_pending': '1'}, conn=conn)
|
repository.set_index_meta({f'note_vectors_pending:{parsed.note_id}': '1' for parsed in prepared}, conn=conn)
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
"""Synthetic, isolated exact-search comparison; does not access the user Vault."""
|
||||||
|
import heapq
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
import random
|
||||||
|
import sqlite3
|
||||||
|
import statistics
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import sqlite_vec
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||||
|
from app.retrieval import space_index
|
||||||
|
from app.retrieval.routed_vectors import RemoteEmbeddings, _unit_vector
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
rng = random.Random(42)
|
||||||
|
count, dimensions = 4000, 384
|
||||||
|
with tempfile.TemporaryDirectory(prefix='notes-vec-bench-') as temporary:
|
||||||
|
conn = sqlite3.connect(Path(temporary) / 'vectors.db')
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
conn.enable_load_extension(True)
|
||||||
|
sqlite_vec.load(conn)
|
||||||
|
conn.enable_load_extension(False)
|
||||||
|
conn.execute('CREATE TABLE blocks(block_id TEXT PRIMARY KEY, embedding_local_only INTEGER)')
|
||||||
|
conn.execute('CREATE TABLE routed_block_vectors(space_id TEXT,block_id TEXT,dimensions INTEGER,vector TEXT,PRIMARY KEY(space_id,dimensions,block_id))')
|
||||||
|
vectors = [_unit_vector([rng.uniform(-1, 1) for _ in range(dimensions)], dimensions) for _ in range(count)]
|
||||||
|
conn.executemany('INSERT INTO blocks VALUES (?,0)', [(str(i),) for i in range(count)])
|
||||||
|
conn.executemany('INSERT INTO routed_block_vectors VALUES (?,?,?,?)', [('benchmark', str(i), dimensions, json.dumps(v)) for i, v in enumerate(vectors)])
|
||||||
|
start = time.perf_counter()
|
||||||
|
space_index.ensure(conn, 'benchmark', dimensions)
|
||||||
|
migration_ms = (time.perf_counter() - start) * 1000
|
||||||
|
conn.commit()
|
||||||
|
query = vectors[0]
|
||||||
|
batch = RemoteEmbeddings('benchmark', dimensions, [query])
|
||||||
|
def legacy():
|
||||||
|
def hits():
|
||||||
|
for row in conn.execute('SELECT block_id,vector FROM routed_block_vectors'):
|
||||||
|
vector = _unit_vector(json.loads(row[1]), dimensions)
|
||||||
|
yield row[0], max(0., min(1., math.fsum(a*b for a,b in zip(query,vector))))
|
||||||
|
return heapq.nlargest(20, hits(), key=lambda hit:hit[1])
|
||||||
|
def native():
|
||||||
|
return [(hit.id,hit.score) for hit in space_index.search(conn,batch,20)]
|
||||||
|
measurements = {}
|
||||||
|
results = {}
|
||||||
|
for name, operation in [('python_json_scan', legacy), ('sqlite_vec',native)]:
|
||||||
|
elapsed = []
|
||||||
|
for _ in range(5):
|
||||||
|
start = time.perf_counter()
|
||||||
|
results[name] = operation()
|
||||||
|
elapsed.append((time.perf_counter()-start)*1000)
|
||||||
|
measurements[name] = {'median_ms':statistics.median(elapsed), 'samples_ms':elapsed}
|
||||||
|
assert [hit[0] for hit in results['python_json_scan']] == [hit[0] for hit in results['sqlite_vec']]
|
||||||
|
print(json.dumps({'blocks':count,'dimensions':dimensions,'top_k':20,'migration_ms':migration_ms,
|
||||||
|
'same_top_k':True,'measurements':measurements},indent=2))
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -66,6 +66,52 @@ async def seed():
|
|||||||
return apple, banana
|
return apple, banana
|
||||||
|
|
||||||
|
|
||||||
|
def test_native_spaces_isolate_dimensions_and_reuse_without_json_scan(runtime, monkeypatch):
|
||||||
|
from app.retrieval import space_index
|
||||||
|
async def scenario():
|
||||||
|
apple, banana = await seed()
|
||||||
|
ids = [b.block_id for note in (apple, banana) for b in note.blocks]
|
||||||
|
conn = connect()
|
||||||
|
try:
|
||||||
|
with transaction(conn):
|
||||||
|
routed_vectors.store_remote(conn, ids, routed_vectors.RemoteEmbeddings('space-a', 4, [[1., 0., 0., 0.]] * len(ids)))
|
||||||
|
assert conn.execute('SELECT COUNT(DISTINCT dimensions) FROM routed_block_vectors').fetchone()[0] == 2
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
# A new connection uses the persistent native index, without reading vector JSON.
|
||||||
|
def forbidden(*args, **kwargs):
|
||||||
|
raise AssertionError('query decoded stored JSON')
|
||||||
|
monkeypatch.setattr(space_index.json, 'loads', forbidden)
|
||||||
|
hits = await routed_vectors.search_remote('apple orchard', top_k=2, strict=True)
|
||||||
|
assert len(hits) == 2
|
||||||
|
assert hits[0].id == apple.blocks[0].block_id
|
||||||
|
asyncio.run(scenario())
|
||||||
|
|
||||||
|
|
||||||
|
def test_legacy_vectors_migrate_without_document_embedding(runtime):
|
||||||
|
from app.retrieval import space_index
|
||||||
|
async def scenario():
|
||||||
|
apple, banana = await seed()
|
||||||
|
conn = connect()
|
||||||
|
table = space_index.table_name('space-a', 3)
|
||||||
|
try:
|
||||||
|
with transaction(conn):
|
||||||
|
conn.execute(f'DROP TRIGGER {table}_delete')
|
||||||
|
conn.execute(f'DROP TRIGGER {table}_update')
|
||||||
|
conn.execute(f'DROP TABLE {table}')
|
||||||
|
conn.execute('ALTER TABLE routed_block_vectors RENAME TO saved_vectors')
|
||||||
|
conn.execute('CREATE TABLE routed_block_vectors(space_id TEXT,block_id TEXT REFERENCES blocks(block_id) ON DELETE CASCADE,dimensions INTEGER,vector TEXT,PRIMARY KEY(space_id,block_id))')
|
||||||
|
conn.execute('INSERT INTO routed_block_vectors SELECT * FROM saved_vectors')
|
||||||
|
conn.execute('DROP TABLE saved_vectors')
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
runtime.calls.clear()
|
||||||
|
hits = await routed_vectors.search_remote('apple orchard', top_k=2, strict=True)
|
||||||
|
assert len(hits) == 2
|
||||||
|
assert runtime.calls == [['apple orchard']]
|
||||||
|
asyncio.run(scenario())
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("outcome", ["api", "api_failure", "missing_space"])
|
@pytest.mark.parametrize("outcome", ["api", "api_failure", "missing_space"])
|
||||||
def test_benchmark_reports_actual_embedding_and_fallback(runtime, outcome):
|
def test_benchmark_reports_actual_embedding_and_fallback(runtime, outcome):
|
||||||
from app.benchmarks import service
|
from app.benchmarks import service
|
||||||
|
|||||||
@@ -5,6 +5,33 @@ from app.config import get_settings
|
|||||||
from app.services import index_service, workspace_service
|
from app.services import index_service, workspace_service
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_new_note_does_not_rebuild_existing_notes(monkeypatch):
|
||||||
|
from app.services import note_service
|
||||||
|
async def scenario():
|
||||||
|
await note_service.create_note(title='Existing', markdown='Keep existing vectors', folder=None, tags=[])
|
||||||
|
calls = []
|
||||||
|
original = index_service.prepare_note_index
|
||||||
|
async def record(parsed, **kwargs):
|
||||||
|
calls.append(parsed.file_path)
|
||||||
|
return await original(parsed, **kwargs)
|
||||||
|
async def forbidden(*args, **kwargs):
|
||||||
|
raise AssertionError('full rebuild should not run')
|
||||||
|
monkeypatch.setattr(index_service, 'prepare_note_index', record)
|
||||||
|
monkeypatch.setattr(index_service, 'rebuild', forbidden)
|
||||||
|
path = get_settings().vault_path / 'external.md'
|
||||||
|
path.write_text('# External\n\nNew content', encoding='utf-8')
|
||||||
|
try:
|
||||||
|
await workspace_service.refresh_workspace_tree()
|
||||||
|
await index_service._background_task
|
||||||
|
assert calls == ['external.md']
|
||||||
|
assert not index_service.get_status().vector_refresh_required
|
||||||
|
await workspace_service.open_workspace(None)
|
||||||
|
assert calls == ['external.md']
|
||||||
|
finally:
|
||||||
|
await index_service.shutdown()
|
||||||
|
asyncio.run(scenario())
|
||||||
|
|
||||||
|
|
||||||
def test_open_returns_before_vectors_and_deduplicates_background(monkeypatch):
|
def test_open_returns_before_vectors_and_deduplicates_background(monkeypatch):
|
||||||
async def scenario():
|
async def scenario():
|
||||||
started, release = asyncio.Event(), asyncio.Event()
|
started, release = asyncio.Event(), asyncio.Event()
|
||||||
|
|||||||
@@ -37,6 +37,7 @@
|
|||||||
- [前端构建分块优化开发说明](development/前端构建分块优化开发说明.md)
|
- [前端构建分块优化开发说明](development/前端构建分块优化开发说明.md)
|
||||||
|
|
||||||
- [工作区后台索引与保存开发说明](development/工作区后台索引与保存开发说明.md)
|
- [工作区后台索引与保存开发说明](development/工作区后台索引与保存开发说明.md)
|
||||||
|
- [模型隔离向量索引与增量登记](development/模型隔离向量索引与增量登记.md)
|
||||||
- [Mermaid 预览与缩放开发说明](development/Mermaid预览与缩放开发说明.md)
|
- [Mermaid 预览与缩放开发说明](development/Mermaid预览与缩放开发说明.md)
|
||||||
- [扩展安装持久化与社区包开发说明](development/扩展安装持久化与社区包开发说明.md)
|
- [扩展安装持久化与社区包开发说明](development/扩展安装持久化与社区包开发说明.md)
|
||||||
- [模型上下文管理](development/模型上下文管理.md)
|
- [模型上下文管理](development/模型上下文管理.md)
|
||||||
|
|||||||
@@ -202,6 +202,7 @@ RunCancelled
|
|||||||
## 2026-09-06:后台索引补充
|
## 2026-09-06:后台索引补充
|
||||||
|
|
||||||
- `POST /api/workspace/open` 返回可使用的 WorkspaceSnapshot,不等待向量推理。
|
- `POST /api/workspace/open` 返回可使用的 WorkspaceSnapshot,不等待向量推理。
|
||||||
|
- 外部新增文件登记后按笔记持久化后台向量任务,不再因此设置全库重建标记;已存在的全库待处理标记仍继续执行。模型空间与维度的持久化 sqlite-vec 索引从已有向量转换,接口响应结构不变。实现与性能验证见 [模型隔离向量索引与增量登记](../development/模型隔离向量索引与增量登记.md)。
|
||||||
- `PATCH /api/notes/{note_id}` 成功代表正文、元数据和 FTS 已保存;后台向量失败不撤销这次保存。
|
- `PATCH /api/notes/{note_id}` 成功代表正文、元数据和 FTS 已保存;后台向量失败不撤销这次保存。
|
||||||
- `GET /api/index/status` 新增 `vector_refresh_required: boolean`,表示工作区或笔记存在向量待处理标记。该字段不是进度百分比;任务失败时也可为 true。
|
- `GET /api/index/status` 新增 `vector_refresh_required: boolean`,表示工作区或笔记存在向量待处理标记。该字段不是进度百分比;任务失败时也可为 true。
|
||||||
- `pending_jobs` 返回真实未完成索引任务数(包含运行中),不再固定为 0。全库待重建或运行中的全量重建计一个任务;逐笔记刷新按待处理标记计数。`running_jobs` 返回当前运行数。失败后保留的待重建标记仍计入未完成数。
|
- `pending_jobs` 返回真实未完成索引任务数(包含运行中),不再固定为 0。全库待重建或运行中的全量重建计一个任务;逐笔记刷新按待处理标记计数。`running_jobs` 返回当前运行数。失败后保留的待重建标记仍计入未完成数。
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
{
|
||||||
|
"blocks": 4000,
|
||||||
|
"dimensions": 384,
|
||||||
|
"top_k": 20,
|
||||||
|
"migration_ms": 1329.5076999929734,
|
||||||
|
"same_top_k": true,
|
||||||
|
"measurements": {
|
||||||
|
"python_json_scan": {
|
||||||
|
"median_ms": 1289.638800022658,
|
||||||
|
"samples_ms": [
|
||||||
|
1289.638800022658,
|
||||||
|
1238.6701999930665,
|
||||||
|
1115.6752999813762,
|
||||||
|
1436.7559000093024,
|
||||||
|
1833.9683999947738
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"sqlite_vec": {
|
||||||
|
"median_ms": 19.673499977216125,
|
||||||
|
"samples_ms": [
|
||||||
|
29.709700000239536,
|
||||||
|
17.876600002637133,
|
||||||
|
35.13619999284856,
|
||||||
|
19.673499977216125,
|
||||||
|
18.28439999371767
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# 模型隔离向量索引与增量登记
|
||||||
|
|
||||||
|
## 目标与实现
|
||||||
|
|
||||||
|
向量仍保存在本地 `app.db`,重启复用。`routed_block_vectors` 主键升级为 `(space_id, dimensions, block_id)`,同维度不同模型、同模型不同维度均独立存储。每个空间建立以模型标识和维度的 SHA-256 命名的 `vec0` 虚拟表,动态表名不包含厂商输入。
|
||||||
|
|
||||||
|
首次访问旧空间时,在事务中将已有 JSON 向量转换为归一化 float32 索引,不重新调用 Embedding 模型。随后检索由 sqlite-vec 执行精确 KNN,避免每次在 Python 中解码所有向量、计算点积。该实现是精确搜索,不是 ANN;依据归一化向量的欧氏距离换算余弦分数。参见 [sqlite-vec KNN 文档](https://alexgarcia.xyz/sqlite-vec/features/knn.html)。
|
||||||
|
|
||||||
|
覆盖检查和 KNN 使用同一事务。向量更新、删除及笔记级联删除通过触发器清理派生索引,新向量与原始向量在同一保存点写入。索引缺失或不完整仍明确失败/按已有策略回退,不混入其他模型结果。本地专用笔记按 `local_only` 元数据过滤,不同空间的结果继续使用 RRF 融合。
|
||||||
|
|
||||||
|
数据库搜索及旧空间转换在线程中运行,不阻塞异步服务事件循环。首次转换仍会占用 SQLite 写事务;超大库应进一步评估迁移耗时。新增空间保留原空间索引,暂不自动清理历史空间。
|
||||||
|
|
||||||
|
## 外部新增文件
|
||||||
|
|
||||||
|
文件树检测到新增 Markdown 后先登记元数据与全文索引,然后为每条新笔记持久化 `note_vectors_pending:<note_id>`,逐笔记在后台计算。不再因新增文件设置全库重建标记;已存在的全库待处理标记仍保留,以免跳过之前未完成的任务。
|
||||||
|
|
||||||
|
新增文件在推理期间再次变化时校验快照并重试,只同步该笔记。写入失败保留待处理标记。手动“重建全部”仍执行全库重建,正常重新打开已完成的工作区不再入队。
|
||||||
|
|
||||||
|
## 验证
|
||||||
|
|
||||||
|
- 隔离数据库运行后端全套测试:637 项通过;最终覆盖检查优化另运行检索及后台工作区回归,96 项通过。
|
||||||
|
- 新增用例验证同模型不同维度并存、重新连接复用且查询不解码向量 JSON、旧表迁移只计算查询向量,以及外部新增文件不触发全量重建。
|
||||||
|
- `backend/scripts/vector-index-benchmark.py` 使用临时数据库、固定随机种子,比较 4000 条 384 维向量的 top-20,旧新路径结果顺序一致。5 次采样中位数:Python 扫描约 1289.64 ms,sqlite-vec 约 19.67 ms;首次转换约 1329.51 ms。该结果只测向量检索,不包含查询 Embedding、重排与 HTTP 耗时,不代表端到端加速比例。
|
||||||
|
- 原始结果:[2026-09-06-sqlite-vec-search.json](performance/2026-09-06-sqlite-vec-search.json)。基准可用后端虚拟环境 Python 直接执行上述脚本,不读写用户 Vault。
|
||||||
Reference in New Issue
Block a user