fix: stabilize background operations and large embedding results

backend/tests/test_agent_core.py

@@ -112,7 +112,7 @@ def test_permission_confirmation_resumes_agent() -> None:
                     break
 
         assert request_id is not None
-        assert container.agent.resolve_permission(
+        assert await container.agent.resolve_permission(
             created.run_id, request_id, "allow_once"
         )
         completed = await container.agent.wait(created.run_id)
@@ -370,8 +370,9 @@ def test_cancelling_permission_wait_cancels_run() -> None:
         )
 
         async with asyncio.timeout(2):
-            while container.agent.get_run(created.run_id).status != AgentRunStatus.waiting_permission:
-                await asyncio.sleep(0)
+            async for event in container.agent.events(created.run_id):
+                if event.event == AgentEventType.permission_required:
+                    break
 
         cancelled = await container.agent.cancel(created.run_id)
         await container.agent.wait(created.run_id)

backend/tests/test_index_activity.py

@@ -0,0 +1,68 @@
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+from app.retrieval import activity
+from app.services import index_service
+
+
+@pytest.fixture(autouse=True)
+def reset_activity(monkeypatch):
+    for field in ('active', 'completed', 'failed', 'cancelled'):
+        monkeypatch.setattr(activity, field, 0)
+    monkeypatch.setattr(index_service, '_active_job_id', None)
+    monkeypatch.setattr(index_service, '_active_scope', None)
+    monkeypatch.setattr(index_service, '_last_error', None)
+    monkeypatch.setattr(index_service.repository, 'stats', lambda: {'notes': 16, 'blocks': 3787})
+    monkeypatch.setattr(index_service.repository, 'get_index_meta', lambda: {})
+
+
+def test_pending_rebuild_and_incremental_jobs(monkeypatch):
+    meta = {'workspace_vectors_pending': '1', 'note_vectors_pending:a': '1', 'note_vectors_pending:b': '1'}
+    monkeypatch.setattr(index_service.repository, 'get_index_meta', lambda: meta)
+    status = index_service.get_status()
+    assert status.vector_refresh_required and status.pending_jobs == 1
+    assert status.running_jobs == 0
+    monkeypatch.setattr(index_service, '_active_job_id', 'job_test')
+    monkeypatch.setattr(index_service, '_active_scope', 'all')
+    status = index_service.get_status()
+    assert (status.status, status.pending_jobs, status.running_jobs) == ('running', 1, 1)
+    monkeypatch.setattr(index_service, '_active_scope', 'note')
+    assert index_service.get_status().pending_jobs == 2
+    meta.pop('workspace_vectors_pending')
+    assert index_service.get_status().pending_jobs == 2
+    monkeypatch.setattr(index_service, '_active_job_id', None)
+    monkeypatch.setattr(index_service, '_last_error', 'failed')
+    status = index_service.get_status()
+    assert (status.status, status.pending_jobs, status.running_jobs) == ('failed', 2, 0)
+    meta.clear()
+    assert index_service.get_status().pending_jobs == 0
+
+
+def test_search_activity_covers_completion_failure_and_cancellation():
+    async def scenario():
+        gate = asyncio.Event()
+
+        @activity.track_search
+        async def search(_self, request):
+            await gate.wait()
+            if request.query == 'fail':
+                raise ValueError('failure')
+            return 'ok'
+
+        tasks = [asyncio.create_task(search(None, SimpleNamespace(mode=mode, query=query)))
+                 for mode, query in [('vector', 'ok'), ('hybrid', 'fail'), ('vector', 'cancel'), ('fts', 'ok')]]
+        await asyncio.sleep(0)
+        assert index_service.get_status().active_searches == 3
+        tasks[2].cancel()
+        with pytest.raises(asyncio.CancelledError):
+            await tasks[2]
+        gate.set()
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        assert results[0] == results[3] == 'ok'
+        status = index_service.get_status()
+        assert (status.active_searches, status.completed_searches, status.failed_searches, status.cancelled_searches) == (0, 1, 1, 1)
+        assert status.pending_jobs == 0
+
+    asyncio.run(scenario())

backend/tests/test_local_models.py

@@ -12,6 +12,42 @@ from app.local_models.runtime import Runtime
 from app.providers.base import ProviderError
 
 
+@pytest.mark.parametrize('threaded', [False, True])
+def test_large_embedding_result_crosses_pipe_limit_without_truncation(monkeypatch, tmp_path, threaded):
+    import app.local_models.runtime as module
+    import app.local_models.process as process_module
+    from app.local_models.protocol import response_lines
+    vector = [0.012345678901234567] * 384
+    result = {'result': [vector] * 2111}
+    assert len(json.dumps(result).encode()) > 16 * 1024 * 1024
+    assert max(map(len, response_lines(result, 'embedding'))) < 16 * 1024 * 1024
+    worker = tmp_path / 'large_worker.py'
+    protocol_dir = Path(module.__file__).parent
+    worker.write_text(
+        'import sys,json\n'
+        f'sys.path.insert(0, {str(protocol_dir)!r})\n'
+        'from protocol import response_lines\n'
+        'request=json.load(sys.stdin)\n'
+        'vector=[0.012345678901234567]*384\n'
+        'for line in response_lines({"result":[vector]*len(request["payload"]["texts"])}, "embedding"):\n'
+        ' sys.stdout.write(line)\n', encoding='utf-8')
+    monkeypatch.setattr(module, 'read_state', lambda key: {'status': 'installed'})
+    monkeypatch.setattr(module, 'interpreter', lambda *_: Path(sys.executable))
+    original_async = asyncio.create_subprocess_exec
+    original_threaded = process_module.ThreadedProcess
+
+    async def spawn(*args, **kwargs):
+        if threaded:
+            raise NotImplementedError
+        return await original_async(sys.executable, str(worker), **kwargs)
+
+    monkeypatch.setattr(module.asyncio, 'create_subprocess_exec', spawn)
+    monkeypatch.setattr(process_module, 'ThreadedProcess',
+                        lambda args, **kwargs: original_threaded((sys.executable, str(worker)), **kwargs))
+    actual = asyncio.run(Runtime().infer('bekko', 'embedding', {'texts': ['test'] * 2111}))
+    assert actual == result['result']
+
+
 def test_download_resumes_partial_and_checks_digest(monkeypatch):
     payload = b'verified-model-weights'
     entry = {'path':'model.safetensors','size':len(payload),'hash':hashlib.sha256(payload).hexdigest(),

backend/tests/test_operation_logs.py

@@ -0,0 +1,151 @@
+import asyncio
+import json
+import logging
+import threading
+from pathlib import Path
+
+from app.operation_logs import LogStore, ApplicationLogHandler, get_store, log_event, shutdown_logging
+
+
+def test_logs_persist_filter_cursor_and_retention(tmp_path):
+    store = LogStore(tmp_path / 'logs.db', retain=3)
+    try:
+        for i in range(6):
+            store.emit('ERROR' if i % 2 else 'INFO', 'vectors', 'embedding.failed', {'run_id': f'run_{i}'})
+        store.queue.join()
+        first = store.query(limit=2)
+        assert len(first['items']) == 2 and first['next_cursor']
+        assert len(store.query(before=first['next_cursor'])['items']) == 1
+        assert len(store.query(level='ERROR')['items']) == 2
+        assert len(store.query(q='run_5')['items']) == 1
+        assert not store.query(source='tasks')['items']
+    finally:
+        store.close()
+    reopened = LogStore(tmp_path / 'logs.db', retain=3)
+    try:
+        assert len(reopened.query()['items']) == 3
+    finally:
+        reopened.close()
+
+
+def test_logs_exclude_content_and_legacy_exception_messages():
+    try:
+        log_event('vectors', 'embedding.failed', level='ERROR',
+                  error=ValueError('private note and secret'), model='embedding-v1',
+                  prompt='private note', arguments={'api_key': 'secret'}, api_key='secret')
+        handler = ApplicationLogHandler()
+        record = logging.LogRecord('app.sample', logging.ERROR, __file__, 1,
+                                   'private note and secret %s', ('credentials',), None)
+        handler.emit(record)
+        handler.emit(record)  # a logger propagated to another installed handler
+        store = get_store()
+        store.queue.join()
+        data = json.dumps(store.query())
+        assert len(store.query()['items']) == 2
+        assert 'private note' not in data and 'credentials' not in data and 'api_key' not in data
+        assert 'ValueError' in data and 'embedding-v1' in data
+    finally:
+        shutdown_logging()
+
+
+def test_http_log_correlates_task_operations_without_body():
+    import httpx
+    from app.main import app
+    async def scenario():
+        async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url='http://test') as client:
+            response = await client.post('/api/tasks', json={'title': 'private title'})
+            assert response.status_code == 200
+            rid = response.headers['x-request-id']
+            store = get_store()
+            await asyncio.to_thread(store.queue.join)
+            result = (await client.get('/api/logs', params={'q': rid})).json()
+            assert len(result['items']) >= 2
+            assert 'private title' not in json.dumps(result)
+            assert any(item['event'] == 'task.created' for item in result['items'])
+            assert (await client.get('/api/logs', params={'limit': 201})).status_code == 422
+    try:
+        asyncio.run(scenario())
+    finally:
+        shutdown_logging()
+
+
+def test_trace_writer_batches_off_loop_and_survives_cancel():
+    from app.agent.async_trace import AsyncTraceWriter
+    started, release = threading.Event(), threading.Event()
+    class Repository:
+        def write_batch(self, jobs):
+            assert threading.current_thread() is not threading.main_thread()
+            started.set()
+            assert release.wait(2)
+            self.jobs = jobs
+    async def scenario():
+        repo = Repository()
+        writer = AsyncTraceWriter(repo)
+        pending = asyncio.create_task(writer.submit('save', 'snapshot'))
+        while not started.is_set():
+            await asyncio.sleep(.001)
+        pending.cancel()
+        writer.worker.cancel()  # simultaneous application shutdown
+        await asyncio.sleep(.005)
+        assert not pending.done()
+        release.set()
+        assert await asyncio.wait_for(pending, 2) is True
+        assert repo.jobs == [('save', ('snapshot',))]
+        assert writer.queue.empty()
+    asyncio.run(scenario())
+
+
+def test_trace_write_failure_is_reported_and_next_submission_recovers():
+    from app.agent.async_trace import AsyncTraceWriter
+    class Repository:
+        fail = True
+        def write_batch(self, jobs):
+            if self.fail:
+                self.fail = False
+                raise OSError('disk unavailable')
+    async def scenario():
+        import pytest
+        writer = AsyncTraceWriter(Repository())
+        with pytest.raises(OSError):
+            await writer.submit('save', 'first')
+        assert not await writer.submit('save', 'second')
+        await writer.queue.join()
+    asyncio.run(scenario())
+
+
+def test_log_write_failure_does_not_stall_queue(tmp_path, monkeypatch):
+    store = LogStore(tmp_path / 'failed.db')
+    try:
+        def broken():
+            raise OSError('disk unavailable')
+        monkeypatch.setattr(store, '_connect', broken)
+        store.emit('ERROR', 'vectors', 'embedding.failed', {'duration_ms': float('nan')})
+        store.queue.join()
+        assert store.failed == 1
+    finally:
+        store.close()
+
+
+def test_cancelled_task_write_keeps_its_slot_until_commit():
+    from app.services.task_service import write_in_background
+    started, release = threading.Event(), threading.Event()
+    order = []
+    def first():
+        started.set()
+        assert release.wait(2)
+        order.append('first')
+    async def scenario():
+        import pytest
+        pending = asyncio.create_task(write_in_background(first))
+        while not started.is_set():
+            await asyncio.sleep(.001)
+        pending.cancel()
+        second = asyncio.create_task(write_in_background(lambda: order.append('second')))
+        await asyncio.sleep(.01)
+        assert order == []
+        release.set()
+        with pytest.raises(asyncio.CancelledError):
+            await pending
+        await second
+        assert order == ['first', 'second']
+    asyncio.run(scenario())