Complete phase two benchmarks, plot previews and static export workflow

backend/scripts/agent-task-stress.py

@@ -96,11 +96,13 @@ async def main(output):
                 assert all(sequences)
                 recovered = AgentRuntime(container.providers, container.tools, container.permissions,
                                          trace_repository=runtime.trace_repository)
-                assert all(recovered.get_run(run_id).status == AgentRunStatus.completed for run_id in ids)
+                recovery_started = time.perf_counter()
+                assert await asyncio.to_thread(lambda: all(recovered.get_run(run_id).status == AgentRunStatus.completed for run_id in ids))
+                recovery_read_ms = (time.perf_counter() - recovery_started) * 1000
                 assert not any(record.subscribers for record in runtime._records.values())
                 return {"concurrency": concurrency, "runs": len(ids), "latency": stats(durations),
                         "completed": statuses.count('completed'), "ordered_events_and_replay": True,
-                        "terminal_recovery": True, "retained_records": len(runtime._records)}
+                        "terminal_recovery": True, "recovery_read_ms": round(recovery_read_ms,2), "retained_records": len(runtime._records)}
             save('agent_tool_runs', await measured(batch))
 
         # Hold model calls so all 200 records remain active while testing admission.

backend/scripts/phase2-context.py

@@ -0,0 +1,50 @@
+"""Two bounded live calls for context summary + answer; never changes saved config."""
+import argparse, asyncio, json, sys
+from pathlib import Path
+sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
+async def main(args):
+    from app.container import container
+    from app.contracts import ModelRequest, ModelContextPolicy
+    from app.providers.factory import ProviderFactory
+    from app.providers.context_budget import prepare_context, estimate
+    from app.providers.base import ProviderError
+    from app.services.usage_service import connection
+    provider=container.providers.get(args.provider)
+    model=provider.config.default_model
+    request=ModelRequest(provider_id=args.provider,model=model,max_tokens=1024,messages=[
+        {'role':'user','content':'项目事实：笔记保存在 Vault，导出使用点击时的快照。'*50},
+        {'role':'assistant','content':'已记录。'}, {'role':'user','content':'请保持中文。'},
+        {'role':'assistant','content':'好的。'}, {'role':'user','content':'笔记保存在什么地方？一句话回答。'}])
+    original=request.model_dump()
+    config=provider.config.model_copy(deep=True)
+    config.context_policies=[ModelContextPolicy(model=model,context_window=8192,output_reserve=512,threshold=.1,mode='detect')]
+    calls=0
+    async def complete(value):
+        nonlocal calls
+        calls+=1
+        return await provider.adapter.complete(value)
+    results={'model':model,'configured_test_window':8192,'vendor_max_context_tested':False}
+    try:
+        try: await prepare_context(request,config,complete)
+        except ProviderError as exc: results['detect']={'error_code':exc.code,'network_calls':calls}
+        config.context_policies[0].mode='compress'
+        config.context_policies[0].prompt='把以下历史资料压缩成一句中文，只保留笔记存储位置和导出快照规则。'
+        prepared=await asyncio.wait_for(prepare_context(request,config,complete),60)
+        turn=await asyncio.wait_for(complete(prepared),60)
+        results['compression']={'passed':'vault' in (turn.text or '').lower(),'before_estimate':estimate(request),
+            'after_estimate':estimate(prepared),'archive_unchanged':request.model_dump()==original,'network_calls':calls,
+            'answer_input_tokens':turn.input_tokens,'answer_output_tokens':turn.output_tokens}
+        with connection() as conn:
+            rows=[json.loads(row[0]) for row in conn.execute('SELECT counters_json FROM model_usage WHERE provider_id=?',(args.provider,))]
+        results['observed_provider_cache']={'requests':len(rows),'reporting_requests':sum(x.get('cache_hit_tokens') is not None for x in rows),
+            'positive_hit_requests':sum((x.get('cache_hit_tokens') or 0)>0 for x in rows),
+            'positive_miss_requests':sum((x.get('cache_miss_tokens') or 0)>0 for x in rows),
+            'scope':'provider reported usage across this isolated acceptance session; not deterministic cache control'}
+    finally:
+        args.output.write_text(json.dumps(results,ensure_ascii=False,indent=2),encoding='utf-8')
+        print(json.dumps(results,ensure_ascii=False))
+        await container.agent.shutdown();container.mcp_servers.shutdown();container.plugins.shutdown()
+if __name__=='__main__':
+    p=argparse.ArgumentParser();p.add_argument('--provider',required=True);p.add_argument('--output',type=Path,required=True);p.add_argument('--execute',action='store_true');args=p.parse_args()
+    if not args.execute:p.error('--execute required; two requests use existing quota')
+    asyncio.run(main(args))

backend/scripts/phase2-integration.py

@@ -0,0 +1,56 @@
+"""Explicit isolated Demo: retrieval → read → three tasks, then real read-only MCP.
+
+Approves only this run's tasks.write tickets. Requires the quality fixture Vault.
+Calls public API contracts; never writes completion state into SQLite.
+"""
+import argparse, json, time
+from pathlib import Path
+from urllib.request import Request,urlopen
+
+def main(args):
+    def api(path,body=None):
+        req=Request(args.base_url+'/api'+path,data=json.dumps(body).encode() if body is not None else None,
+                    headers={'Content-Type':'application/json'})
+        with urlopen(req,timeout=60) as response:return json.load(response)
+    directory=args.data_dir.resolve()
+    if not (directory/'vault/.phase2-fixture').exists():raise SystemExit('Isolated fixture Vault required')
+    core=json.loads((directory/'benchmarks/rag-phase2-v1.json').read_text(encoding='utf-8'))
+    note=next(c for c in core['cases'] if c['case_id']=='deadlock-0')['expected_note_ids'][0]
+    cases=[{'case_id':'retrieval-tasks','prompt':f'按顺序执行：1. 用 rag.search 搜索“死锁”，明确使用 mode=fts；2. 用 notes.read 读取笔记 {note}；3. 根据内容用 tasks.create 分别创建且仅创建三个任务，标题严格为“验收-互斥条件”、“验收-循环等待”、“验收-锁顺序”；4. 总结死锁条件并引用搜索来源。请不要调用其他工具，不重复创建。',
+        'allowed_tools':['rag.search','notes.read','tasks.create'],
+        'expected_tools':[{'name':'rag.search','arguments':{'query':'死锁','mode':'fts'}},{'name':'notes.read','arguments':{'note_id':note}}]+
+            [{'name':'tasks.create','arguments':{'title':title}} for title in ['验收-互斥条件','验收-循环等待','验收-锁顺序']],
+        'citation_required':True,'tasks_created':3,'output_contains':['死锁'],'tags':['rag','notes','tasks','permissions']}]
+    tools=api('/tools')['items']; mcp=next((t for t in tools if t['name'].endswith('.web_search') and t['name'].startswith('mcp.')),None)
+    if mcp:
+        cases.append({'case_id':'mcp-search','prompt':f'调用一次 {mcp["name"]}，query 严格使用 Python official documentation tutorial。根据工具真实返回给出一句总结。',
+            'allowed_tools':[mcp['name']], 'expected_tools':[{'name':mcp['name'],'arguments':{'query':'Python official documentation tutorial'}}], 'tags':['mcp','real-network']})
+    dataset={'dataset_id':'agent-integration-v1','kind':'agent','version':'1.0.0','description':'Isolated phase2 cross-module Demo; existing MCP binding captured','cases':cases}
+    (directory/'benchmarks/agent-integration-v1.json').write_text(json.dumps(dataset,ensure_ascii=False,indent=2),encoding='utf-8')
+    providers=api('/providers')['items']; provider=next(p for p in providers if p['enabled'] and p['provider_type']!='mock')
+    run=api('/benchmarks/agent/runs',{'dataset_id':dataset['dataset_id'],'provider_id':provider['provider_id'],'model':provider['default_model'],
+        'max_steps':10,'timeout_seconds':150,'token_budget':10000,'allow_network':True})
+    approved=set();deadline=time.monotonic()+360
+    while run['status'] in ['queued','running']:
+        if time.monotonic()>deadline:
+            api('/benchmarks/runs/'+run['run_id']+'/cancel',{});raise RuntimeError('Demo deadline')
+        active=run['config_snapshot'].get('active_agent_run_id')
+        if active:
+            trace=api('/agent/runs/'+active+'/trace')
+            for event in trace['items']:
+                data=event['data'];ticket=data.get('request_id')
+                if event['event']=='PermissionRequired' and data.get('permission')=='tasks.write' and ticket not in approved:
+                    api('/agent/runs/'+active+'/permissions/'+ticket,{'decision':'allow_once'});approved.add(ticket)
+        time.sleep(.3);run=api('/benchmarks/runs/'+run['run_id'])
+    report=api('/benchmarks/runs/'+run['run_id']+'/report')
+    report['permission_approvals']=len(approved)
+    report['mcp_present']=bool(mcp)
+    report['tasks']= [{'task_id':t['task_id'],'title':t['title']} for t in api('/tasks')['items'] if t['title'].startswith('验收-')]
+    args.output.write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding='utf-8')
+    print(json.dumps({'metrics':report['metrics'],'cases':report['cases'],'permission_approvals':len(approved)},ensure_ascii=False))
+
+if __name__=='__main__':
+    p=argparse.ArgumentParser();p.add_argument('--base-url',default='http://127.0.0.1:8017');p.add_argument('--data-dir',type=Path,required=True);p.add_argument('--output',type=Path,required=True);p.add_argument('--execute',action='store_true');args=p.parse_args()
+    if not args.execute:p.error('--execute required; creates three tasks only in the isolated fixture application')
+    if args.base_url not in {'http://127.0.0.1:8017','http://localhost:8017'}:p.error('Use the isolated local acceptance server on port 8017')
+    main(args)

backend/scripts/phase2-live.py

@@ -0,0 +1,64 @@
+"""Bounded real protocol/Agent checks using existing configuration; no secret output.
+
+Requires --execute; at most 5 direct model requests plus one 4-case Agent dataset
+(6 steps and 6000 tokens per case). No provisioning or external writes.
+"""
+import argparse, asyncio, json, sys
+from pathlib import Path
+from time import perf_counter
+sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
+
+async def main(args):
+    from app.container import container
+    from app.contracts import ModelRequest, AgentBenchmarkRequest
+    from app.providers.base import ProviderError
+    from app.benchmarks import agent,service
+    provider=container.providers.get(args.provider)
+    adapter=provider.adapter; model=provider.config.default_model
+    results={'provider_id':args.provider,'protocol':provider.config.provider_type.value,'model':model,'checks':{},
+        'unconfigured_protocols':['openai_responses','anthropic_messages','ollama'],
+        'not_tested':['provider_context_limit','cache_hit_miss','context_compression'],'max_direct_requests':5}
+    def request(prompt='Reply OK only.', **changes):
+        return ModelRequest(provider_id=args.provider,model=model,messages=[{'role':'user','content':prompt}],max_tokens=128,**changes)
+    async def check(name, fn):
+        started=perf_counter()
+        try: results['checks'][name]=await asyncio.wait_for(fn(),60)
+        except ProviderError as exc: results['checks'][name]={'passed':False,'error_code':exc.code}
+        except Exception as exc: results['checks'][name]={'passed':False,'error_type':type(exc).__name__}
+        results['checks'][name]['elapsed_ms']=round((perf_counter()-started)*1000,2)
+        print(name,results['checks'][name],flush=True)
+    async def discover():
+        models=await adapter.list_models();return {'passed':bool(models),'count':len(models)}
+    async def complete():
+        turn=await adapter.complete(request());return {'passed':bool(turn.text),'input_tokens':turn.input_tokens,'output_tokens':turn.output_tokens}
+    async def stream():
+        events=[e async for e in adapter.stream(request())]
+        names=[e.event.value for e in events]
+        return {'passed':names.count('Done')==1 and 'TextDelta' in names,'events':sorted(set(names)),
+                'usage_reported':'Usage' in names,'reasoning_observed':'ThinkingDelta' in names}
+    async def cancel():
+        iterator=adapter.stream(request('List the integers 1 through 1000.'))
+        first=await anext(iterator);started=perf_counter();await iterator.aclose()
+        return {'passed':True,'first_event':first.event.value,'close_ms':(perf_counter()-started)*1000,
+                'scope':'client stream resource close; provider billing cessation not observable'}
+    async def invalid_model():
+        try: await adapter.complete(request().model_copy(update={'model':'notesagent-nonexistent-acceptance-model'}))
+        except ProviderError as exc:return {'passed':True,'error_code':exc.code}
+        return {'passed':False,'reason':'provider accepted unknown model'}
+    try:
+        for name,fn in [('discovery',discover),('normal_chat',complete),('stream_usage_reasoning',stream),('stream_cancel',cancel),('error_mapping',invalid_model)]:await check(name,fn)
+        created=await agent.create_run(AgentBenchmarkRequest(dataset_id='agent-core-v1',provider_id=args.provider,model=model))
+        await service.wait_for_run(created.run_id)
+        results['agent']=service.get_report(created.run_id).model_dump(mode='json')
+        print('agent',results['agent']['metrics'],flush=True)
+        servers=container.mcp_servers.list()
+        results['mcp']=[{'server_id':s.server_id,'name':s.name,'state':s.status.value if hasattr(s.status,'value') else s.status} for s in servers]
+    finally:
+        args.output.parent.mkdir(parents=True,exist_ok=True)
+        args.output.write_text(json.dumps(results,ensure_ascii=False,indent=2,default=str),encoding='utf-8')
+        await container.agent.shutdown();container.mcp_servers.shutdown();container.plugins.shutdown()
+
+if __name__=='__main__':
+    p=argparse.ArgumentParser();p.add_argument('--provider',required=True);p.add_argument('--output',type=Path,required=True);p.add_argument('--execute',action='store_true');args=p.parse_args()
+    if not args.execute:p.error('--execute required; uses existing provider quota')
+    asyncio.run(main(args))

backend/scripts/phase2-quality.py

@@ -0,0 +1,69 @@
+"""Reproducible real-model quality run in an explicitly isolated APP_DATA_DIR.
+
+Uses the application index/benchmark services. Never injects vectors or completion rows.
+Existing local weights/runtime must be configured; inference does not download models.
+"""
+import argparse
+import asyncio
+import hashlib
+import json
+import os
+import sys
+from pathlib import Path
+from datetime import datetime, timezone
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+async def main(args):
+    from app.config import get_settings
+    from app.knowledge.parser import parse_note
+    from app.services import index_service
+    from app.benchmarks import service
+    from app.contracts import IndexRebuildRequest, RAGRunRequest
+    settings=get_settings()
+    if not all(os.getenv(name) for name in ['APP_DATA_DIR','APP_DB_PATH','APP_VAULT_PATH']):
+        raise SystemExit('Explicit isolated APP_DATA_DIR/APP_DB_PATH/APP_VAULT_PATH required')
+    fixture=Path(__file__).resolve().parents[1]/'data/benchmarks/corpus/phase2-v1.json'
+    payload=json.loads(fixture.read_text(encoding='utf-8')); cases=[]
+    settings.vault_path.mkdir(parents=True,exist_ok=True)
+    if list(settings.vault_path.glob('*.md')) and not (settings.vault_path/'.phase2-fixture').exists():
+        raise SystemExit('Refusing to overwrite a non-fixture vault')
+    now=datetime(2026,9,7,tzinfo=timezone.utc)
+    for note in payload['notes']:
+        name=note['id']+'.md'; markdown='# '+note['title']+'\n\n'+note['text']+'\n'
+        (settings.vault_path/name).write_text(markdown,encoding='utf-8')
+        parsed=parse_note(markdown=markdown,file_path=name,folder='',created_at=now,updated_at=now)
+        for index,query in enumerate(note['queries']):
+            cases.append({'case_id':note['id']+'-'+str(index),'query':query,'expected_note_ids':[parsed.note_id],
+                'expected_block_ids':[parsed.blocks[-1].block_id],'citation_required':True,
+                'tags':['keyword' if index==0 else 'paraphrase',note['id']]})
+    (settings.vault_path/'.phase2-fixture').touch()
+    dataset={'dataset_id':'rag-phase2-v1','kind':'rag','version':payload['version'],'description':payload['description'],'cases':cases}
+    settings.benchmark_datasets_path.mkdir(parents=True,exist_ok=True)
+    (settings.benchmark_datasets_path/'rag-phase2-v1.json').write_text(json.dumps(dataset,ensure_ascii=False,indent=2),encoding='utf-8')
+    if not args.reuse_index:
+        job=await index_service.rebuild(IndexRebuildRequest())
+        if job.status != 'completed': raise RuntimeError('Index did not complete: '+str(job.status))
+    from app import repository
+    expected_blocks = {bid for case in cases for bid in case['expected_block_ids']}
+    if {hit.block_id for hit in repository.get_block_hits(list(expected_blocks))} != expected_blocks:
+        raise RuntimeError('Frozen corpus does not match the index; rerun without --reuse-index')
+    reports={}
+    for label, mode, fusion, rerank, k in [('fts','fts','rrf',False,60),('vector','vector','rrf',False,60),
+        ('hybrid-weighted','hybrid','weighted',False,60),('hybrid-rrf','hybrid','rrf',False,60),
+        ('hybrid-rerank','hybrid','rrf',True,60),('rrf-k20','hybrid','rrf',False,20)]:
+        run=await service.create_rag_run(RAGRunRequest(dataset_id='rag-phase2-v1',modes=[mode],repeat=2,
+            retrieval={'top_k':5,'fusion':fusion,'rerank':rerank,'rrf_k':k,'rerank_candidates':20},
+            metadata={'corpus_sha256':hashlib.sha256(fixture.read_bytes()).hexdigest(),'split':'development; no held-out production claim'}))
+        await service.wait_for_run(run.run_id)
+        reports[label]=service.get_report(run.run_id).model_dump(mode='json')
+        print(label, json.dumps(reports[label]['metrics']),flush=True)
+        args.output.parent.mkdir(parents=True,exist_ok=True)
+        args.output.write_text(json.dumps(reports,ensure_ascii=False,indent=2),encoding='utf-8')
+    args.output.parent.mkdir(parents=True,exist_ok=True)
+    args.output.write_text(json.dumps(reports,ensure_ascii=False,indent=2),encoding='utf-8')
+    from app.container import container
+    await container.agent.shutdown(); container.mcp_servers.shutdown(); container.plugins.shutdown()
+
+if __name__=='__main__':
+    parser=argparse.ArgumentParser(); parser.add_argument('--output',type=Path,required=True); parser.add_argument('--reuse-index',action='store_true')
+    asyncio.run(main(parser.parse_args()))