Merge remote-tracking branch 'origin/main' into feat/export-service
# Conflicts: # .gitignore # backend/app/main.py
This commit is contained in:
@@ -112,7 +112,7 @@ def test_permission_confirmation_resumes_agent() -> None:
|
||||
break
|
||||
|
||||
assert request_id is not None
|
||||
assert container.agent.resolve_permission(
|
||||
assert await container.agent.resolve_permission(
|
||||
created.run_id, request_id, "allow_once"
|
||||
)
|
||||
completed = await container.agent.wait(created.run_id)
|
||||
@@ -370,8 +370,9 @@ def test_cancelling_permission_wait_cancels_run() -> None:
|
||||
)
|
||||
|
||||
async with asyncio.timeout(2):
|
||||
while container.agent.get_run(created.run_id).status != AgentRunStatus.waiting_permission:
|
||||
await asyncio.sleep(0)
|
||||
async for event in container.agent.events(created.run_id):
|
||||
if event.event == AgentEventType.permission_required:
|
||||
break
|
||||
|
||||
cancelled = await container.agent.cancel(created.run_id)
|
||||
await container.agent.wait(created.run_id)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.retrieval import activity
|
||||
from app.services import index_service
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_activity(monkeypatch):
|
||||
for field in ('active', 'completed', 'failed', 'cancelled'):
|
||||
monkeypatch.setattr(activity, field, 0)
|
||||
monkeypatch.setattr(index_service, '_active_job_id', None)
|
||||
monkeypatch.setattr(index_service, '_active_scope', None)
|
||||
monkeypatch.setattr(index_service, '_last_error', None)
|
||||
monkeypatch.setattr(index_service.repository, 'stats', lambda: {'notes': 16, 'blocks': 3787})
|
||||
monkeypatch.setattr(index_service.repository, 'get_index_meta', lambda: {})
|
||||
|
||||
|
||||
def test_pending_rebuild_and_incremental_jobs(monkeypatch):
|
||||
meta = {'workspace_vectors_pending': '1', 'note_vectors_pending:a': '1', 'note_vectors_pending:b': '1'}
|
||||
monkeypatch.setattr(index_service.repository, 'get_index_meta', lambda: meta)
|
||||
status = index_service.get_status()
|
||||
assert status.vector_refresh_required and status.pending_jobs == 1
|
||||
assert status.running_jobs == 0
|
||||
monkeypatch.setattr(index_service, '_active_job_id', 'job_test')
|
||||
monkeypatch.setattr(index_service, '_active_scope', 'all')
|
||||
status = index_service.get_status()
|
||||
assert (status.status, status.pending_jobs, status.running_jobs) == ('running', 1, 1)
|
||||
monkeypatch.setattr(index_service, '_active_scope', 'note')
|
||||
assert index_service.get_status().pending_jobs == 2
|
||||
meta.pop('workspace_vectors_pending')
|
||||
assert index_service.get_status().pending_jobs == 2
|
||||
monkeypatch.setattr(index_service, '_active_job_id', None)
|
||||
monkeypatch.setattr(index_service, '_last_error', 'failed')
|
||||
status = index_service.get_status()
|
||||
assert (status.status, status.pending_jobs, status.running_jobs) == ('failed', 2, 0)
|
||||
meta.clear()
|
||||
assert index_service.get_status().pending_jobs == 0
|
||||
|
||||
|
||||
def test_search_activity_covers_completion_failure_and_cancellation():
|
||||
async def scenario():
|
||||
gate = asyncio.Event()
|
||||
|
||||
@activity.track_search
|
||||
async def search(_self, request):
|
||||
await gate.wait()
|
||||
if request.query == 'fail':
|
||||
raise ValueError('failure')
|
||||
return 'ok'
|
||||
|
||||
tasks = [asyncio.create_task(search(None, SimpleNamespace(mode=mode, query=query)))
|
||||
for mode, query in [('vector', 'ok'), ('hybrid', 'fail'), ('vector', 'cancel'), ('fts', 'ok')]]
|
||||
await asyncio.sleep(0)
|
||||
assert index_service.get_status().active_searches == 3
|
||||
tasks[2].cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await tasks[2]
|
||||
gate.set()
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
assert results[0] == results[3] == 'ok'
|
||||
status = index_service.get_status()
|
||||
assert (status.active_searches, status.completed_searches, status.failed_searches, status.cancelled_searches) == (0, 1, 1, 1)
|
||||
assert status.pending_jobs == 0
|
||||
|
||||
asyncio.run(scenario())
|
||||
@@ -12,6 +12,42 @@ from app.local_models.runtime import Runtime
|
||||
from app.providers.base import ProviderError
|
||||
|
||||
|
||||
@pytest.mark.parametrize('threaded', [False, True])
|
||||
def test_large_embedding_result_crosses_pipe_limit_without_truncation(monkeypatch, tmp_path, threaded):
|
||||
import app.local_models.runtime as module
|
||||
import app.local_models.process as process_module
|
||||
from app.local_models.protocol import response_lines
|
||||
vector = [0.012345678901234567] * 384
|
||||
result = {'result': [vector] * 2111}
|
||||
assert len(json.dumps(result).encode()) > 16 * 1024 * 1024
|
||||
assert max(map(len, response_lines(result, 'embedding'))) < 16 * 1024 * 1024
|
||||
worker = tmp_path / 'large_worker.py'
|
||||
protocol_dir = Path(module.__file__).parent
|
||||
worker.write_text(
|
||||
'import sys,json\n'
|
||||
f'sys.path.insert(0, {str(protocol_dir)!r})\n'
|
||||
'from protocol import response_lines\n'
|
||||
'request=json.load(sys.stdin)\n'
|
||||
'vector=[0.012345678901234567]*384\n'
|
||||
'for line in response_lines({"result":[vector]*len(request["payload"]["texts"])}, "embedding"):\n'
|
||||
' sys.stdout.write(line)\n', encoding='utf-8')
|
||||
monkeypatch.setattr(module, 'read_state', lambda key: {'status': 'installed'})
|
||||
monkeypatch.setattr(module, 'interpreter', lambda *_: Path(sys.executable))
|
||||
original_async = asyncio.create_subprocess_exec
|
||||
original_threaded = process_module.ThreadedProcess
|
||||
|
||||
async def spawn(*args, **kwargs):
|
||||
if threaded:
|
||||
raise NotImplementedError
|
||||
return await original_async(sys.executable, str(worker), **kwargs)
|
||||
|
||||
monkeypatch.setattr(module.asyncio, 'create_subprocess_exec', spawn)
|
||||
monkeypatch.setattr(process_module, 'ThreadedProcess',
|
||||
lambda args, **kwargs: original_threaded((sys.executable, str(worker)), **kwargs))
|
||||
actual = asyncio.run(Runtime().infer('bekko', 'embedding', {'texts': ['test'] * 2111}))
|
||||
assert actual == result['result']
|
||||
|
||||
|
||||
def test_download_resumes_partial_and_checks_digest(monkeypatch):
|
||||
payload = b'verified-model-weights'
|
||||
entry = {'path':'model.safetensors','size':len(payload),'hash':hashlib.sha256(payload).hexdigest(),
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from app.operation_logs import LogStore, ApplicationLogHandler, get_store, log_event, shutdown_logging
|
||||
|
||||
|
||||
def test_logs_persist_filter_cursor_and_retention(tmp_path):
|
||||
store = LogStore(tmp_path / 'logs.db', retain=3)
|
||||
try:
|
||||
for i in range(6):
|
||||
store.emit('ERROR' if i % 2 else 'INFO', 'vectors', 'embedding.failed', {'run_id': f'run_{i}'})
|
||||
store.queue.join()
|
||||
first = store.query(limit=2)
|
||||
assert len(first['items']) == 2 and first['next_cursor']
|
||||
assert len(store.query(before=first['next_cursor'])['items']) == 1
|
||||
assert len(store.query(level='ERROR')['items']) == 2
|
||||
assert len(store.query(q='run_5')['items']) == 1
|
||||
assert not store.query(source='tasks')['items']
|
||||
finally:
|
||||
store.close()
|
||||
reopened = LogStore(tmp_path / 'logs.db', retain=3)
|
||||
try:
|
||||
assert len(reopened.query()['items']) == 3
|
||||
finally:
|
||||
reopened.close()
|
||||
|
||||
|
||||
def test_logs_exclude_content_and_legacy_exception_messages():
|
||||
try:
|
||||
log_event('vectors', 'embedding.failed', level='ERROR',
|
||||
error=ValueError('private note and secret'), model='embedding-v1',
|
||||
prompt='private note', arguments={'api_key': 'secret'}, api_key='secret')
|
||||
handler = ApplicationLogHandler()
|
||||
record = logging.LogRecord('app.sample', logging.ERROR, __file__, 1,
|
||||
'private note and secret %s', ('credentials',), None)
|
||||
handler.emit(record)
|
||||
handler.emit(record) # a logger propagated to another installed handler
|
||||
store = get_store()
|
||||
store.queue.join()
|
||||
data = json.dumps(store.query())
|
||||
assert len(store.query()['items']) == 2
|
||||
assert 'private note' not in data and 'credentials' not in data and 'api_key' not in data
|
||||
assert 'ValueError' in data and 'embedding-v1' in data
|
||||
finally:
|
||||
shutdown_logging()
|
||||
|
||||
|
||||
def test_http_log_correlates_task_operations_without_body():
|
||||
import httpx
|
||||
from app.main import app
|
||||
async def scenario():
|
||||
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url='http://test') as client:
|
||||
response = await client.post('/api/tasks', json={'title': 'private title'})
|
||||
assert response.status_code == 200
|
||||
rid = response.headers['x-request-id']
|
||||
store = get_store()
|
||||
await asyncio.to_thread(store.queue.join)
|
||||
result = (await client.get('/api/logs', params={'q': rid})).json()
|
||||
assert len(result['items']) >= 2
|
||||
assert 'private title' not in json.dumps(result)
|
||||
assert any(item['event'] == 'task.created' for item in result['items'])
|
||||
assert (await client.get('/api/logs', params={'limit': 201})).status_code == 422
|
||||
try:
|
||||
asyncio.run(scenario())
|
||||
finally:
|
||||
shutdown_logging()
|
||||
|
||||
|
||||
def test_trace_writer_batches_off_loop_and_survives_cancel():
|
||||
from app.agent.async_trace import AsyncTraceWriter
|
||||
started, release = threading.Event(), threading.Event()
|
||||
class Repository:
|
||||
def write_batch(self, jobs):
|
||||
assert threading.current_thread() is not threading.main_thread()
|
||||
started.set()
|
||||
assert release.wait(2)
|
||||
self.jobs = jobs
|
||||
async def scenario():
|
||||
repo = Repository()
|
||||
writer = AsyncTraceWriter(repo)
|
||||
pending = asyncio.create_task(writer.submit('save', 'snapshot'))
|
||||
while not started.is_set():
|
||||
await asyncio.sleep(.001)
|
||||
pending.cancel()
|
||||
writer.worker.cancel() # simultaneous application shutdown
|
||||
await asyncio.sleep(.005)
|
||||
assert not pending.done()
|
||||
release.set()
|
||||
assert await asyncio.wait_for(pending, 2) is True
|
||||
assert repo.jobs == [('save', ('snapshot',))]
|
||||
assert writer.queue.empty()
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_trace_write_failure_is_reported_and_next_submission_recovers():
|
||||
from app.agent.async_trace import AsyncTraceWriter
|
||||
class Repository:
|
||||
fail = True
|
||||
def write_batch(self, jobs):
|
||||
if self.fail:
|
||||
self.fail = False
|
||||
raise OSError('disk unavailable')
|
||||
async def scenario():
|
||||
import pytest
|
||||
writer = AsyncTraceWriter(Repository())
|
||||
with pytest.raises(OSError):
|
||||
await writer.submit('save', 'first')
|
||||
assert not await writer.submit('save', 'second')
|
||||
await writer.queue.join()
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_log_write_failure_does_not_stall_queue(tmp_path, monkeypatch):
|
||||
store = LogStore(tmp_path / 'failed.db')
|
||||
try:
|
||||
def broken():
|
||||
raise OSError('disk unavailable')
|
||||
monkeypatch.setattr(store, '_connect', broken)
|
||||
store.emit('ERROR', 'vectors', 'embedding.failed', {'duration_ms': float('nan')})
|
||||
store.queue.join()
|
||||
assert store.failed == 1
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def test_cancelled_task_write_keeps_its_slot_until_commit():
|
||||
from app.services.task_service import write_in_background
|
||||
started, release = threading.Event(), threading.Event()
|
||||
order = []
|
||||
def first():
|
||||
started.set()
|
||||
assert release.wait(2)
|
||||
order.append('first')
|
||||
async def scenario():
|
||||
import pytest
|
||||
pending = asyncio.create_task(write_in_background(first))
|
||||
while not started.is_set():
|
||||
await asyncio.sleep(.001)
|
||||
pending.cancel()
|
||||
second = asyncio.create_task(write_in_background(lambda: order.append('second')))
|
||||
await asyncio.sleep(.01)
|
||||
assert order == []
|
||||
release.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await pending
|
||||
await second
|
||||
assert order == ['first', 'second']
|
||||
asyncio.run(scenario())
|
||||
@@ -66,6 +66,161 @@ async def seed():
|
||||
return apple, banana
|
||||
|
||||
|
||||
def test_native_spaces_isolate_dimensions_and_reuse_without_json_scan(runtime, monkeypatch):
|
||||
from app.retrieval import space_index
|
||||
async def scenario():
|
||||
apple, banana = await seed()
|
||||
ids = [b.block_id for note in (apple, banana) for b in note.blocks]
|
||||
conn = connect()
|
||||
try:
|
||||
with transaction(conn):
|
||||
routed_vectors.store_remote(conn, ids, routed_vectors.RemoteEmbeddings('space-a', 4, [[1., 0., 0., 0.]] * len(ids)))
|
||||
assert conn.execute('SELECT COUNT(DISTINCT dimensions) FROM routed_block_vectors').fetchone()[0] == 2
|
||||
finally:
|
||||
conn.close()
|
||||
# A new connection uses the persistent native index, without reading vector JSON.
|
||||
def forbidden(*args, **kwargs):
|
||||
raise AssertionError('query decoded stored JSON')
|
||||
monkeypatch.setattr(space_index.json, 'loads', forbidden)
|
||||
hits = await routed_vectors.search_remote('apple orchard', top_k=2, strict=True)
|
||||
assert len(hits) == 2
|
||||
assert hits[0].id == apple.blocks[0].block_id
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_legacy_vectors_migrate_without_document_embedding(runtime):
|
||||
from app.retrieval import space_index
|
||||
async def scenario():
|
||||
apple, banana = await seed()
|
||||
conn = connect()
|
||||
table = space_index.table_name('space-a', 3)
|
||||
try:
|
||||
with transaction(conn):
|
||||
conn.execute(f'DROP TRIGGER {table}_delete')
|
||||
conn.execute(f'DROP TRIGGER {table}_update')
|
||||
conn.execute(f'DROP TABLE {table}')
|
||||
conn.execute('ALTER TABLE routed_block_vectors RENAME TO saved_vectors')
|
||||
conn.execute('CREATE TABLE routed_block_vectors(space_id TEXT,block_id TEXT REFERENCES blocks(block_id) ON DELETE CASCADE,dimensions INTEGER,vector TEXT,PRIMARY KEY(space_id,block_id))')
|
||||
conn.execute('INSERT INTO routed_block_vectors SELECT * FROM saved_vectors')
|
||||
conn.execute('DROP TABLE saved_vectors')
|
||||
finally:
|
||||
conn.close()
|
||||
runtime.calls.clear()
|
||||
hits = await routed_vectors.search_remote('apple orchard', top_k=2, strict=True)
|
||||
assert len(hits) == 2
|
||||
assert runtime.calls == [['apple orchard']]
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.parametrize('partitioned', [False, True])
|
||||
def test_concurrent_first_search_serializes_migration_and_warm_search_is_read_only(runtime, monkeypatch, partitioned):
|
||||
import threading
|
||||
from app.retrieval import space_index
|
||||
async def scenario():
|
||||
await seed()
|
||||
table = space_index.table_name('space-a', 3)
|
||||
conn = connect()
|
||||
try:
|
||||
with transaction(conn):
|
||||
conn.execute(f'DROP TRIGGER {table}_delete')
|
||||
conn.execute(f'DROP TRIGGER {table}_update')
|
||||
conn.execute(f'DROP TABLE {table}')
|
||||
finally:
|
||||
conn.close()
|
||||
entered, release, second = threading.Event(), threading.Event(), threading.Event()
|
||||
original_ensure, original_prepare = space_index.ensure, space_index.prepare
|
||||
calls = []
|
||||
def ensure(*args):
|
||||
calls.append(1)
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
return original_ensure(*args)
|
||||
def prepare(*args):
|
||||
if entered.is_set():
|
||||
second.set()
|
||||
return original_prepare(*args)
|
||||
monkeypatch.setattr(space_index, 'ensure', ensure)
|
||||
monkeypatch.setattr(space_index, 'prepare', prepare)
|
||||
batch = routed_vectors.RemoteEmbeddings('space-a', 3, [[1., 0., 0.]])
|
||||
async def search():
|
||||
if entered.is_set():
|
||||
second.set()
|
||||
await routed_vectors._prepare_indexes([batch])
|
||||
if partitioned:
|
||||
return await asyncio.to_thread(routed_vectors._search_partitions, {False: batch}, {False}, 2, True)
|
||||
return await asyncio.to_thread(routed_vectors._search_space, batch, 2, True)
|
||||
tasks = []
|
||||
try:
|
||||
tasks.append(asyncio.create_task(search()))
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
tasks.append(asyncio.create_task(search()))
|
||||
assert await asyncio.to_thread(second.wait, 5)
|
||||
release.set()
|
||||
first, other = await asyncio.gather(*tasks)
|
||||
assert first == other and len(first) == 2
|
||||
assert len(calls) == 1
|
||||
# Prepared indexes are reusable even with SQLite query_only enforced.
|
||||
original_connect = routed_vectors.connect
|
||||
def read_only():
|
||||
connection = original_connect()
|
||||
connection.execute('PRAGMA query_only=ON')
|
||||
return connection
|
||||
monkeypatch.setattr(routed_vectors, 'connect', read_only)
|
||||
assert await search() == first
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.parametrize('cancel_search', [False, True])
|
||||
def test_save_waits_for_migration_even_when_search_is_cancelled(runtime, monkeypatch, cancel_search):
|
||||
import threading
|
||||
from app.retrieval import space_index
|
||||
async def scenario():
|
||||
apple, _ = await seed()
|
||||
table = space_index.table_name('space-a', 3)
|
||||
conn = connect()
|
||||
try:
|
||||
with transaction(conn):
|
||||
conn.execute(f'DROP TRIGGER {table}_delete')
|
||||
conn.execute(f'DROP TRIGGER {table}_update')
|
||||
conn.execute(f'DROP TABLE {table}')
|
||||
finally:
|
||||
conn.close()
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
original = space_index.ensure
|
||||
def slow(*args):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
return original(*args)
|
||||
monkeypatch.setattr(space_index, 'ensure', slow)
|
||||
# Keep the subsequent vector job queued; test saving and its durable marker.
|
||||
monkeypatch.setattr(index_service, 'schedule_workspace_rebuild', lambda: None)
|
||||
query = asyncio.create_task(routed_vectors.search_remote('apple orchard', top_k=2, strict=True))
|
||||
save = None
|
||||
try:
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
if cancel_search:
|
||||
query.cancel()
|
||||
save = asyncio.create_task(note_service.update_note(apple.note_id, markdown='Saved during migration', defer_vectors=True))
|
||||
await asyncio.sleep(0.02)
|
||||
assert not save.done()
|
||||
release.set()
|
||||
saved = await asyncio.wait_for(save, 5)
|
||||
assert saved.markdown == 'Saved during migration'
|
||||
assert (await note_service.get_note(apple.note_id)).markdown == saved.markdown
|
||||
assert repository.get_index_meta()[f'note_vectors_pending:{apple.note_id}'] == '1'
|
||||
# Query may observe the saved revision's pending index, but saving must succeed.
|
||||
result = (await asyncio.gather(query, return_exceptions=True))[0]
|
||||
if cancel_search:
|
||||
assert isinstance(result, asyncio.CancelledError)
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.gather(*([query, save] if save else [query]), return_exceptions=True)
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("outcome", ["api", "api_failure", "missing_space"])
|
||||
def test_benchmark_reports_actual_embedding_and_fallback(runtime, outcome):
|
||||
from app.benchmarks import service
|
||||
|
||||
@@ -5,6 +5,33 @@ from app.config import get_settings
|
||||
from app.services import index_service, workspace_service
|
||||
|
||||
|
||||
def test_external_new_note_does_not_rebuild_existing_notes(monkeypatch):
|
||||
from app.services import note_service
|
||||
async def scenario():
|
||||
await note_service.create_note(title='Existing', markdown='Keep existing vectors', folder=None, tags=[])
|
||||
calls = []
|
||||
original = index_service.prepare_note_index
|
||||
async def record(parsed, **kwargs):
|
||||
calls.append(parsed.file_path)
|
||||
return await original(parsed, **kwargs)
|
||||
async def forbidden(*args, **kwargs):
|
||||
raise AssertionError('full rebuild should not run')
|
||||
monkeypatch.setattr(index_service, 'prepare_note_index', record)
|
||||
monkeypatch.setattr(index_service, 'rebuild', forbidden)
|
||||
path = get_settings().vault_path / 'external.md'
|
||||
path.write_text('# External\n\nNew content', encoding='utf-8')
|
||||
try:
|
||||
await workspace_service.refresh_workspace_tree()
|
||||
await index_service._background_task
|
||||
assert calls == ['external.md']
|
||||
assert not index_service.get_status().vector_refresh_required
|
||||
await workspace_service.open_workspace(None)
|
||||
assert calls == ['external.md']
|
||||
finally:
|
||||
await index_service.shutdown()
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_open_returns_before_vectors_and_deduplicates_background(monkeypatch):
|
||||
async def scenario():
|
||||
started, release = asyncio.Event(), asyncio.Event()
|
||||
|
||||
Reference in New Issue
Block a user