perf: persist model-isolated sqlite-vec indexes and index new notes incrementally

backend/app/retrieval/routed_vectors.py

@@ -2,15 +2,14 @@
 
 The runtime's model_id is the authoritative space ID (including provider URL,
 endpoint, model and dimensions); equal dimensions alone never imply compatibility.
-This phase uses a lazy, rebuildable SQLite side table instead of a schema migration.
-Search scans only current blocks in one database snapshot and requires complete
-coverage. Cosine ranking costs O(blocks * dimensions) with an O(top_k) heap; this
-small-vault implementation should become a per-space ANN index at larger scale.
+Durable vectors are reused to build per-space/dimension sqlite-vec indexes lazily.
+Native exact KNN avoids Python JSON decoding and dot products on every search.
+Coverage checks and ranking share one transaction.
 """
 
 from __future__ import annotations
 
-import heapq
+import asyncio
 import json
 import logging
 import math
@@ -24,6 +23,7 @@ from app.operation_logs import log_event
 from app.retrieval.vectorstore import VectorHit
 from app.retrieval.provenance import record_embedding
 from app.retrieval.hybrid import rrf_fuse
+from app.retrieval import space_index
 
 logger = logging.getLogger(__name__)
 
@@ -121,9 +121,15 @@ def _ensure_table(conn: sqlite3.Connection) -> None:
             block_id TEXT NOT NULL REFERENCES blocks(block_id) ON DELETE CASCADE,
             dimensions INTEGER NOT NULL CHECK (dimensions > 0),
             vector TEXT NOT NULL,
-            PRIMARY KEY (space_id, block_id)
+            PRIMARY KEY (space_id, dimensions, block_id)
         )
     """)
+    primary = [row[1] for row in sorted(conn.execute('PRAGMA table_info(routed_block_vectors)'), key=lambda row: row[5]) if row[5]]
+    if primary == ['space_id', 'block_id']:
+        conn.execute('CREATE TABLE routed_block_vectors_upgrade (space_id TEXT NOT NULL, block_id TEXT NOT NULL REFERENCES blocks(block_id) ON DELETE CASCADE, dimensions INTEGER NOT NULL CHECK(dimensions>0), vector TEXT NOT NULL, PRIMARY KEY(space_id,dimensions,block_id))')
+        conn.execute('INSERT INTO routed_block_vectors_upgrade SELECT * FROM routed_block_vectors')
+        conn.execute('DROP TABLE routed_block_vectors')
+        conn.execute('ALTER TABLE routed_block_vectors_upgrade RENAME TO routed_block_vectors')
     conn.execute("""
         CREATE INDEX IF NOT EXISTS routed_block_vectors_block_id
         ON routed_block_vectors(block_id)
@@ -149,13 +155,14 @@ def store_remote(
             conn.executemany(
                 """INSERT INTO routed_block_vectors (space_id, block_id, dimensions, vector)
                    VALUES (?, ?, ?, ?)
-                   ON CONFLICT (space_id, block_id) DO UPDATE SET
+                   ON CONFLICT (space_id, dimensions, block_id) DO UPDATE SET
                        dimensions = excluded.dimensions, vector = excluded.vector""",
                 [
                     (batch.space_id, block_id, batch.dimensions, json.dumps(vector, allow_nan=False))
                     for block_id, vector in zip(block_ids, batch.vectors)
                 ],
             )
+            space_index.upsert(conn, block_ids, batch)
         except BaseException:
             conn.execute("ROLLBACK TO routed_vectors_write")
             raise
@@ -183,6 +190,10 @@ async def search_remote(query: str, *, top_k: int, accept_local=False, strict=Fa
     if batch is None:
         return None
 
+    return await asyncio.to_thread(_search_space, batch, top_k, strict)
+
+
+def _search_space(batch, top_k, strict):
     record_embedding(attempted_space={"model_id": batch.space_id, "dimensions": batch.dimensions})
     try:
         conn = connect()
@@ -198,29 +209,8 @@ async def search_remote(query: str, *, top_k: int, accept_local=False, strict=Fa
                     if strict:
                         raise ValueError("semantic index missing")
                     return None
-                rows = conn.execute(
-                    """SELECT b.block_id, r.vector
-                       FROM blocks AS b
-                       LEFT JOIN routed_block_vectors AS r
-                         ON r.block_id = b.block_id AND r.space_id = ? AND r.dimensions = ?
-                       ORDER BY b.block_id""",
-                    (batch.space_id, batch.dimensions),
-                )
-
-                def hits():
-                    for row in rows:
-                        if row["vector"] is None:
-                            raise ValueError("remote space has incomplete block coverage")
-                        vector = _unit_vector(json.loads(row["vector"]), batch.dimensions)
-                        score = math.fsum(a * b for a, b in zip(batch.vectors[0], vector))
-                        yield VectorHit(id=row["block_id"], score=max(0.0, min(1.0, score)))
-
-                try:
-                    result = heapq.nlargest(top_k, hits(), key=lambda hit: hit.score)
-                finally:
-                    # Exceptions may retain the generator/traceback; finalize its
-                    # cursor now so a subsequent rebuild can acquire a write lock.
-                    rows.close()
+                _ensure_table(conn)
+                result = space_index.search(conn, batch, top_k)
                 record_embedding(source=batch.source, model_id=batch.space_id,
                                  dimensions=batch.dimensions, fallback_reason=None)
                 return result
@@ -244,6 +234,10 @@ async def _search_partitioned(query: str, policies: set[bool], *, top_k: int, st
         if batch is None:
             return None
         batches[policy] = batch
+    return await asyncio.to_thread(_search_partitions, batches, policies, top_k, strict)
+
+
+def _search_partitions(batches, policies, top_k, strict):
     conn = connect()
     try:
         with transaction(conn):
@@ -253,23 +247,8 @@ async def _search_partitioned(query: str, policies: set[bool], *, top_k: int, st
                 raise ValueError("embedding policies changed while querying")
             ranked = []
             for policy, batch in batches.items():
-                rows = conn.execute(
-                    "SELECT b.block_id,r.vector FROM blocks b LEFT JOIN routed_block_vectors r "
-                    "ON r.block_id=b.block_id AND r.space_id=? AND r.dimensions=? "
-                    "WHERE b.embedding_local_only=? ORDER BY b.block_id",
-                    (batch.space_id, batch.dimensions, int(policy)),
-                )
-                def hits():
-                    for row in rows:
-                        if row['vector'] is None:
-                            raise ValueError("incomplete policy coverage")
-                        vector = _unit_vector(json.loads(row['vector']), batch.dimensions)
-                        score = math.fsum(a * b for a, b in zip(batch.vectors[0], vector))
-                        yield VectorHit(id=row['block_id'], score=max(0.0, min(1.0, score)))
-                try:
-                    ranked.append(heapq.nlargest(top_k, hits(), key=lambda hit: hit.score))
-                finally:
-                    rows.close()
+                _ensure_table(conn)
+                ranked.append(space_index.search(conn, batch, top_k, policy))
             spaces = [{"source": b.source, "model_id": b.space_id, "dimensions": b.dimensions,
                        "local_only": policy} for policy, b in batches.items()]
             record_embedding(source="mixed" if len({b.source for b in batches.values()}) > 1 else batch.source,

backend/app/retrieval/space_index.py

@@ -0,0 +1,59 @@
+"""Persistent vec0 indexes derived from durable routed vectors, one per space/dimension."""
+import hashlib
+import json
+
+import sqlite_vec
+
+from app.retrieval.vectorstore import VectorHit
+
+
+def table_name(space, dimensions):
+    return 'routed_vec_' + hashlib.sha256(json.dumps([space, dimensions]).encode()).hexdigest()
+
+
+def ensure(conn, space, dimensions):
+    from app.retrieval.routed_vectors import _unit_vector
+    table = table_name(space, dimensions)
+    if conn.execute('SELECT 1 FROM sqlite_master WHERE name=?', (table,)).fetchone():
+        return table
+    if type(dimensions) is not int or not 0 < dimensions <= 8192:
+        raise ValueError('unsupported vector dimensions')
+    conn.execute(f'CREATE VIRTUAL TABLE {table} USING vec0(block_id TEXT PRIMARY KEY, embedding float[{dimensions}], local_only INTEGER)')
+    for row in conn.execute('SELECT r.block_id,r.vector,b.embedding_local_only FROM routed_block_vectors r JOIN blocks b USING(block_id) WHERE r.space_id=? AND r.dimensions=?', (space, dimensions)):
+        conn.execute(f'INSERT INTO {table}(block_id,embedding,local_only) VALUES (?,?,?)',
+                     (row[0], sqlite_vec.serialize_float32(_unit_vector(json.loads(row[1]), dimensions)), row[2]))
+    literal = conn.execute('SELECT quote(?)', (space,)).fetchone()[0]
+    for event in ('DELETE', 'UPDATE'):
+        conn.execute(f'''CREATE TRIGGER {table}_{event.lower()} AFTER {event} ON routed_block_vectors
+            WHEN old.space_id={literal} AND old.dimensions={dimensions}
+            BEGIN DELETE FROM {table} WHERE block_id=old.block_id; END''')
+    return table
+
+
+def upsert(conn, block_ids, batch):
+    from app.retrieval.routed_vectors import _unit_vector
+    table = ensure(conn, batch.space_id, batch.dimensions)
+    for block_id, vector in zip(block_ids, batch.vectors):
+        conn.execute(f'DELETE FROM {table} WHERE block_id=?', (block_id,))
+        conn.execute(f'INSERT INTO {table}(block_id,embedding,local_only) SELECT block_id,?,embedding_local_only FROM blocks WHERE block_id=?',
+                     (sqlite_vec.serialize_float32(_unit_vector(vector, batch.dimensions)), block_id))
+
+
+def search(conn, batch, top_k, policy=None):
+    table = ensure(conn, batch.space_id, batch.dimensions)
+    # Coverage checks stay relational; no JSON decoding or Python dot products on the hot path.
+    where = '' if policy is None else ' AND b.embedding_local_only=?'
+    params = () if policy is None else (int(policy),)
+    missing = conn.execute(f'''SELECT 1 FROM blocks b LEFT JOIN routed_block_vectors r
+        ON r.block_id=b.block_id AND r.space_id=? AND r.dimensions=?
+        WHERE r.block_id IS NULL{where} LIMIT 1''', (batch.space_id, batch.dimensions, *params)).fetchone()
+    expected = conn.execute('SELECT COUNT(*) FROM blocks' + ('' if policy is None else ' WHERE embedding_local_only=?'), params).fetchone()[0]
+    actual = conn.execute(f'SELECT COUNT(*) FROM {table}' + ('' if policy is None else ' WHERE local_only=?'), params).fetchone()[0]
+    if missing or actual != expected:
+        raise ValueError('incomplete vector space coverage')
+    if top_k <= 0:
+        return []
+    rows = conn.execute(f'SELECT block_id,distance FROM {table} WHERE embedding MATCH ? AND k=?'
+                        + ('' if policy is None else ' AND local_only=?'),
+                        (sqlite_vec.serialize_float32(batch.vectors[0]), top_k, *params)).fetchall()
+    return [VectorHit(id=row[0], score=max(0.0, min(1.0, 1 - row[1] ** 2 / 2))) for row in rows]

backend/app/services/index_service.py

@@ -271,6 +271,14 @@ async def _refresh_saved_note(note_id: str) -> None:
             conn = connect()
             try:
                 with transaction(conn):
+                    existing_ids = {row[0] for row in conn.execute('SELECT block_id FROM blocks WHERE note_id=?', (note_id,))}
+                    if existing_ids != {block.block_id for block in parsed.blocks}:
+                        # An external editor changed a newly registered note while inference ran.
+                        # Reconcile that note only; the snapshot check above protects newer saves.
+                        parsed.title = parse_note(markdown=markdown, file_path=record.file_path,
+                            folder=record.folder, tags=record.tags, created_at=record.created_at,
+                            updated_at=record.updated_at, note_id=note_id).title
+                        await index_note(parsed, prepared=prepared, conn=conn)
                     # Write only vectors: metadata and FTS already represent the saved revision.
                     vectors, remote = prepared
                     from app.retrieval.vectorstore import VectorRecord
@@ -278,6 +286,14 @@ async def _refresh_saved_note(note_id: str) -> None:
                     await vector_store.upsert([VectorRecord(id=b.block_id, vector=v)
                         for b, v in zip(parsed.blocks, vectors)], conn=conn)
                     routed_vectors.store_remote(conn, [b.block_id for b in parsed.blocks], remote)
+                    if isinstance(note_service.embedding, LocalEmbedding) and parsed.blocks:
+                        from app.retrieval.space_index import table_name
+                        if remote is None:
+                            raise ApiError(503, 'EMBEDDING_UNAVAILABLE', '笔记已保存，向量计算未完成。')
+                        table = table_name(remote.space_id, remote.dimensions)
+                        missing = conn.execute(f'SELECT 1 FROM blocks b LEFT JOIN {table} v ON v.block_id=b.block_id WHERE b.note_id=? AND v.block_id IS NULL LIMIT 1', (note_id,)).fetchone()
+                        if missing:
+                            raise ApiError(500, 'SEMANTIC_INDEX_WRITE_FAILED', '向量写入未完成，保留待处理标记。')
                     repository.set_index_meta({key: '0'}, conn=conn)
             finally:
                 conn.close()

backend/app/services/workspace_service.py

@@ -161,7 +161,7 @@ async def _register_workspace_files() -> None:
                     created_at=parsed.created_at, updated_at=parsed.updated_at, blocks=parsed.blocks)
                 conn.execute('UPDATE blocks SET embedding_local_only=? WHERE note_id=?', (int(parsed.embedding_local_only), parsed.note_id))
             if prepared:
-                repository.set_index_meta({'workspace_vectors_pending': '1'}, conn=conn)
+                repository.set_index_meta({f'note_vectors_pending:{parsed.note_id}': '1' for parsed in prepared}, conn=conn)
     finally:
         conn.close()
 

backend/scripts/vector-index-benchmark.py

@@ -0,0 +1,64 @@
+"""Synthetic, isolated exact-search comparison; does not access the user Vault."""
+import heapq
+import json
+import math
+import random
+import sqlite3
+import statistics
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+import sqlite_vec
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from app.retrieval import space_index
+from app.retrieval.routed_vectors import RemoteEmbeddings, _unit_vector
+
+
+def main():
+    rng = random.Random(42)
+    count, dimensions = 4000, 384
+    with tempfile.TemporaryDirectory(prefix='notes-vec-bench-') as temporary:
+        conn = sqlite3.connect(Path(temporary) / 'vectors.db')
+        conn.row_factory = sqlite3.Row
+        conn.enable_load_extension(True)
+        sqlite_vec.load(conn)
+        conn.enable_load_extension(False)
+        conn.execute('CREATE TABLE blocks(block_id TEXT PRIMARY KEY, embedding_local_only INTEGER)')
+        conn.execute('CREATE TABLE routed_block_vectors(space_id TEXT,block_id TEXT,dimensions INTEGER,vector TEXT,PRIMARY KEY(space_id,dimensions,block_id))')
+        vectors = [_unit_vector([rng.uniform(-1, 1) for _ in range(dimensions)], dimensions) for _ in range(count)]
+        conn.executemany('INSERT INTO blocks VALUES (?,0)', [(str(i),) for i in range(count)])
+        conn.executemany('INSERT INTO routed_block_vectors VALUES (?,?,?,?)', [('benchmark', str(i), dimensions, json.dumps(v)) for i, v in enumerate(vectors)])
+        start = time.perf_counter()
+        space_index.ensure(conn, 'benchmark', dimensions)
+        migration_ms = (time.perf_counter() - start) * 1000
+        conn.commit()
+        query = vectors[0]
+        batch = RemoteEmbeddings('benchmark', dimensions, [query])
+        def legacy():
+            def hits():
+                for row in conn.execute('SELECT block_id,vector FROM routed_block_vectors'):
+                    vector = _unit_vector(json.loads(row[1]), dimensions)
+                    yield row[0], max(0., min(1., math.fsum(a*b for a,b in zip(query,vector))))
+            return heapq.nlargest(20, hits(), key=lambda hit:hit[1])
+        def native():
+            return [(hit.id,hit.score) for hit in space_index.search(conn,batch,20)]
+        measurements = {}
+        results = {}
+        for name, operation in [('python_json_scan', legacy), ('sqlite_vec',native)]:
+            elapsed = []
+            for _ in range(5):
+                start = time.perf_counter()
+                results[name] = operation()
+                elapsed.append((time.perf_counter()-start)*1000)
+            measurements[name] = {'median_ms':statistics.median(elapsed), 'samples_ms':elapsed}
+        assert [hit[0] for hit in results['python_json_scan']] == [hit[0] for hit in results['sqlite_vec']]
+        print(json.dumps({'blocks':count,'dimensions':dimensions,'top_k':20,'migration_ms':migration_ms,
+                          'same_top_k':True,'measurements':measurements},indent=2))
+        conn.close()
+
+
+if __name__ == '__main__':
+    main()

backend/tests/test_routed_retrieval.py

@@ -66,6 +66,52 @@ async def seed():
     return apple, banana
 
 
+def test_native_spaces_isolate_dimensions_and_reuse_without_json_scan(runtime, monkeypatch):
+    from app.retrieval import space_index
+    async def scenario():
+        apple, banana = await seed()
+        ids = [b.block_id for note in (apple, banana) for b in note.blocks]
+        conn = connect()
+        try:
+            with transaction(conn):
+                routed_vectors.store_remote(conn, ids, routed_vectors.RemoteEmbeddings('space-a', 4, [[1., 0., 0., 0.]] * len(ids)))
+            assert conn.execute('SELECT COUNT(DISTINCT dimensions) FROM routed_block_vectors').fetchone()[0] == 2
+        finally:
+            conn.close()
+        # A new connection uses the persistent native index, without reading vector JSON.
+        def forbidden(*args, **kwargs):
+            raise AssertionError('query decoded stored JSON')
+        monkeypatch.setattr(space_index.json, 'loads', forbidden)
+        hits = await routed_vectors.search_remote('apple orchard', top_k=2, strict=True)
+        assert len(hits) == 2
+        assert hits[0].id == apple.blocks[0].block_id
+    asyncio.run(scenario())
+
+
+def test_legacy_vectors_migrate_without_document_embedding(runtime):
+    from app.retrieval import space_index
+    async def scenario():
+        apple, banana = await seed()
+        conn = connect()
+        table = space_index.table_name('space-a', 3)
+        try:
+            with transaction(conn):
+                conn.execute(f'DROP TRIGGER {table}_delete')
+                conn.execute(f'DROP TRIGGER {table}_update')
+                conn.execute(f'DROP TABLE {table}')
+                conn.execute('ALTER TABLE routed_block_vectors RENAME TO saved_vectors')
+                conn.execute('CREATE TABLE routed_block_vectors(space_id TEXT,block_id TEXT REFERENCES blocks(block_id) ON DELETE CASCADE,dimensions INTEGER,vector TEXT,PRIMARY KEY(space_id,block_id))')
+                conn.execute('INSERT INTO routed_block_vectors SELECT * FROM saved_vectors')
+                conn.execute('DROP TABLE saved_vectors')
+        finally:
+            conn.close()
+        runtime.calls.clear()
+        hits = await routed_vectors.search_remote('apple orchard', top_k=2, strict=True)
+        assert len(hits) == 2
+        assert runtime.calls == [['apple orchard']]
+    asyncio.run(scenario())
+
+
 @pytest.mark.parametrize("outcome", ["api", "api_failure", "missing_space"])
 def test_benchmark_reports_actual_embedding_and_fallback(runtime, outcome):
     from app.benchmarks import service

backend/tests/test_workspace_background.py

@@ -5,6 +5,33 @@ from app.config import get_settings
 from app.services import index_service, workspace_service
 
 
+def test_external_new_note_does_not_rebuild_existing_notes(monkeypatch):
+    from app.services import note_service
+    async def scenario():
+        await note_service.create_note(title='Existing', markdown='Keep existing vectors', folder=None, tags=[])
+        calls = []
+        original = index_service.prepare_note_index
+        async def record(parsed, **kwargs):
+            calls.append(parsed.file_path)
+            return await original(parsed, **kwargs)
+        async def forbidden(*args, **kwargs):
+            raise AssertionError('full rebuild should not run')
+        monkeypatch.setattr(index_service, 'prepare_note_index', record)
+        monkeypatch.setattr(index_service, 'rebuild', forbidden)
+        path = get_settings().vault_path / 'external.md'
+        path.write_text('# External\n\nNew content', encoding='utf-8')
+        try:
+            await workspace_service.refresh_workspace_tree()
+            await index_service._background_task
+            assert calls == ['external.md']
+            assert not index_service.get_status().vector_refresh_required
+            await workspace_service.open_workspace(None)
+            assert calls == ['external.md']
+        finally:
+            await index_service.shutdown()
+    asyncio.run(scenario())
+
+
 def test_open_returns_before_vectors_and_deduplicates_background(monkeypatch):
     async def scenario():
         started, release = asyncio.Event(), asyncio.Event()