Complete phase two benchmarks, plot previews and static export workflow

This commit is contained in:
2026-09-07 02:54:52 +08:00
parent 95095197df
commit 89df10bc4e
59 changed files with 2535 additions and 83 deletions
@@ -0,0 +1,65 @@
<script setup lang="ts">
import { computed, ref, watch, onMounted, onBeforeUnmount } from 'vue'
import { benchmarkService as service, type BenchmarkRun } from '@/services/benchmarkService'
import { listProviders } from '@/services/providerService'
const kind = ref<'rag' | 'agent'>('rag'), dataset = ref(''), error = ref(''), busy = ref(false)
const datasets = ref<Awaited<ReturnType<typeof service.datasets>>>([]), runs = ref<BenchmarkRun[]>([])
const providers = ref<Awaited<ReturnType<typeof listProviders>>>([]), provider = ref(''), model = ref('')
const report = ref<Awaited<ReturnType<typeof service.report>> | null>(null)
const topK = ref(5), rrfK = ref(60), rerank = ref(false)
const fusion = ref<'rrf' | 'weighted'>('rrf')
const metricLabels: Record<string, string> = {
total_cases: '计划样本', evaluated_cases: '已评样本', successful_cases: '运行成功', failed_cases: '运行失败',
task_success_rate: '任务成功率', tool_selection_accuracy: '工具选择准确率', tool_argument_accuracy: '参数准确率',
invalid_tool_call_rate: '无效调用率', average_steps: '平均步骤', average_latency_ms: '平均耗时 (ms)',
token_usage: 'Token 用量', tool_calls: '实际工具调用', expected_calls: '预期工具调用',
hit_at_1: 'Hit@1', hit_at_5: 'Hit@5', recall_at_k: 'Recall@K', mrr: 'MRR', citation_hit_rate: '引用命中率',
p50_latency_ms: 'P50 (ms)', p95_latency_ms: 'P95 (ms)', failure_rate: '运行失败率',
}
const metricGroups = computed(() => {
const metrics = report.value?.metrics ?? {}
const groups = 'task_success_rate' in metrics ? { Agent: metrics } : metrics
return Object.entries(groups).filter(([, value]) => value && typeof value === 'object').map(([name, value]) => ({
name, rows: Object.entries(value as Record<string, unknown>).map(([key, number]) => ({
label: metricLabels[key] ?? key,
value: number === null ? '不适用' : typeof number === 'number' ? Number(number.toFixed(4)).toLocaleString() : String(number),
})),
}))
})
let timer: ReturnType<typeof setTimeout> | undefined, disposed = false
async function loadDatasets() { try { datasets.value = await service.datasets(kind.value); dataset.value = datasets.value[0]?.id ?? '' } catch(e) { error.value = String(e) } }
async function refresh() { try { runs.value = await service.list() } catch(e) { error.value = String(e) } if (!disposed) timer = setTimeout(refresh, 1500) }
watch(kind, loadDatasets)
watch(provider, id => { model.value = providers.value.find(p => p.provider_id === id)?.default_model ?? '' })
async function start() {
error.value = ''; busy.value = true
try { await service.start(kind.value, kind.value === 'agent' ? { dataset_id: dataset.value, provider_id: provider.value, model: model.value, max_steps: 6, timeout_seconds: 90, token_budget: 6000 } : { dataset_id: dataset.value, modes: ['fts','vector','hybrid'], retrieval: { top_k: topK.value, fusion: fusion.value, rrf_k: rrfK.value, rerank: rerank.value } }) }
catch(e) { error.value = String(e) } finally { busy.value = false }
}
async function action(run: BenchmarkRun, cancel = false) { try { if (cancel) await service.cancel(run.id); else report.value = await service.report(run.id) } catch(e) { error.value = String(e) } }
function download() { const url = URL.createObjectURL(new Blob([JSON.stringify(report.value,null,2)], { type:'application/json' })); const a=document.createElement('a'); a.href=url; a.download='benchmark-report.json'; a.click(); setTimeout(()=>URL.revokeObjectURL(url),1000) }
onMounted(async () => { void refresh(); void loadDatasets(); try { providers.value=(await listProviders()).filter(p=>p.enabled); provider.value=providers.value[0]?.provider_id ?? '' } catch(e) { error.value=String(e) } })
onBeforeUnmount(() => { disposed=true; clearTimeout(timer) })
</script>
<template>
<main class="benchmark-page"><h1>Benchmark 评测</h1><p>标准数据集通过真实检索引擎或 Agent Runtime 执行Agent 会使用所选提供商额度需要权限时请打开 Trace 处理</p>
<div class="controls"><label>类型 <select v-model="kind"><option value="rag">RAG</option><option value="agent">Agent</option></select></label>
<label>数据集 <select v-model="dataset"><option v-for="d in datasets" :key="d.id" :value="d.id">{{ d.id }} · {{ d.cases }} 案例</option></select></label>
<template v-if="kind === 'agent'"><label>提供商 <select v-model="provider"><option v-for="p in providers" :key="p.provider_id" :value="p.provider_id">{{ p.name }}</option></select></label><label>模型 <input v-model="model"></label></template>
<template v-else><label>融合 <select v-model="fusion"><option value="rrf">RRF</option><option value="weighted">加权 50/50</option></select></label><label>Top K <input v-model.number="topK" type="number" min="1" max="100"></label><label>RRF K <input v-model.number="rrfK" type="number" min="1"></label><label><input v-model="rerank" type="checkbox">Lexical Reranker</label></template>
<button :disabled="busy || !dataset || (kind === 'agent' && (!provider || !model))" @click="start">运行评测</button></div>
<p v-if="error" role="alert">{{ error }}</p>
<table><thead><tr><th>数据集</th><th>状态</th><th>操作</th></tr></thead><tbody><tr v-for="run in runs" :key="run.id"><td>{{ run.datasetId }}<small>{{ run.id }}</small></td><td>{{ run.status }} {{ run.progress === null ? '' : `${Math.round(run.progress*100)}%` }} {{ run.errorCode }}</td><td><button v-if="['queued','running'].includes(run.status)" @click="action(run,true)">取消</button><button v-else @click="action(run)">查看报告</button><RouterLink v-if="run.agentId" :to="`/agent/runs/${run.agentId}`">Agent Trace</RouterLink></td></tr></tbody></table>
<section v-if="report"><h2>评测报告</h2><button class="button-secondary" @click="download">下载完整 JSON</button>
<div v-for="group in metricGroups" :key="group.name"><h3>{{ group.name }}</h3><dl class="metric-grid"><div v-for="row in group.rows" :key="row.label"><dt>{{ row.label }}</dt><dd>{{ row.value }}</dd></div></dl></div>
<details><summary>冻结配置与逐例证据</summary><pre>{{ JSON.stringify(report,null,2) }}</pre></details></section>
</main>
</template>
<style scoped>
.benchmark-page { padding:24px; overflow:auto; width:100%; } .controls { display:flex; gap:12px; flex-wrap:wrap; } label { display:flex; align-items:center; gap:6px; } input[type=number] { width:80px; } table { width:100%; margin-block:20px; border-collapse:collapse; } td,th { text-align:left; padding:12px; border-bottom:1px solid var(--color-border-default); } small { display:block; } pre { white-space:pre-wrap; overflow-wrap:anywhere; } [role=alert] { color:var(--color-error); }
.metric-grid { display:grid; grid-template-columns:repeat(auto-fit,minmax(150px,1fr)); gap:12px; margin:16px 0; }
.metric-grid > div { padding:16px; border:1px solid var(--color-border-default); border-radius:8px; background:var(--color-surface-primary); }
dt { font-size:13px; color:var(--color-text-secondary); } dd { margin:8px 0 0; font-size:22px; font-weight:600; }
button { padding:6px 12px; border:1px solid var(--color-border-default); border-radius:6px; background:var(--color-surface-primary); cursor:pointer; }
button:disabled { opacity:.5; cursor:default; } td a { margin-left:12px; } input:not([type=checkbox]) { border:1px solid var(--color-border-default); border-radius:6px; padding:6px; }
</style>
@@ -1,6 +1,7 @@
<script setup lang="ts">
import { useEditorStore } from '@/stores/editor'
import { useWorkspaceStore } from '@/stores/workspace'
import ExportDialog from './ExportDialog.vue'
import { computed, ref } from 'vue'
import ActionDialog from '@/components/common/ActionDialog.vue'
import { useActionDialog } from '@/composables/useActionDialog'
@@ -10,6 +11,7 @@ const editorStore = useEditorStore()
const workspaceStore = useWorkspaceStore()
const { actionDialog, resolveAction, askConfirm } = useActionDialog()
const reloadError = ref('')
const exportOpen = ref(false)
const needsRecovery = computed(() => ['conflict', 'external_changed'].includes(editorStore.saveStatus))
const missingFile = computed(() => needsRecovery.value && editorStore.currentFilePath === workspaceStore.activeFilePath && !workspaceStore.activeFile && !workspaceStore.treeRefreshError)
function downloadCopy() {
@@ -43,9 +45,11 @@ const statusText = computed<Record<string, string>>(() => ({
<template>
<header class="editor-header">
<ExportDialog v-if="exportOpen" @close="exportOpen = false" />
<ActionDialog v-if="actionDialog" v-bind="actionDialog" @resolve="resolveAction" />
<div class="file-identity"><strong>{{ workspaceStore.activeFile?.name ?? t('未命名笔记', 'Untitled note') }}</strong><small>{{ workspaceStore.activeFilePath }}</small></div>
<div class="editor-actions">
<button class="button-secondary" @click="exportOpen = true">{{ t('导出', 'Export') }}</button>
<span class="save-status" :class="editorStore.saveStatus">{{ statusText[editorStore.saveStatus] }}</span>
<button v-if="needsRecovery && !missingFile" class="button-secondary" @click="reload">{{ t('重新加载外部版本', 'Reload external version') }}</button>
<span v-if="missingFile" class="save-status conflict">{{ t('原文件已删除或移动', 'Original file deleted or moved') }}</span>
@@ -0,0 +1,56 @@
<script setup lang="ts">
import { ref, onMounted, onBeforeUnmount } from 'vue'
import AppDialog from '@/components/common/AppDialog.vue'
import { useEditorStore } from '@/stores/editor'
import { useThemeStore } from '@/stores/theme'
import { exportService, type ExportFormat, type ExportJob } from '@/services/exportService'
const emit = defineEmits<{ close: [] }>()
const editor = useEditorStore(), theme = useThemeStore()
const format = ref<ExportFormat>('html'), page = ref('A4'), title = ref(true)
const jobs = ref<ExportJob[]>([]), error = ref(''), preparing = ref(false)
let timer: ReturnType<typeof setTimeout> | undefined, disposed = false
let controller: AbortController | undefined
const labels = { queued: '排队中', running: '渲染中', completed: '已完成', failed: '失败', cancelled: '已取消' }
async function refresh() {
try { const value = await exportService.list(); if (!disposed) jobs.value = value } catch (e) { error.value = String(e) }
if (!disposed) timer = setTimeout(refresh, 1500)
}
async function start() {
preparing.value = true; error.value = ''; controller = new AbortController()
const snapshot = editor.content, name = editor.currentFilePath?.split('/').pop()?.replace(/\.md$/i, '') ?? '笔记'
try {
const job = await exportService.create(snapshot, name, format.value, { theme_id: theme.currentThemeId, include_title: title.value, page_size: page.value }, controller.signal, editor.currentFilePath ?? undefined)
if (!disposed) jobs.value.unshift(job)
} catch (e) { error.value = controller.signal.aborted ? '已取消图表准备' : String(e) }
finally { preparing.value = false }
}
async function action(job: ExportJob, download = false) {
try { if (download) await exportService.download(job); else await exportService.cancel(job.id) } catch (e) { error.value = String(e) }
}
onMounted(refresh)
onBeforeUnmount(() => { disposed = true; clearTimeout(timer); controller?.abort() })
</script>
<template>
<AppDialog label="导出笔记" @close="emit('close')"><section class="modal export-modal">
<h2>导出笔记</h2><p>导出点击时的编辑器快照包含未保存修改关闭窗口后后台任务继续运行</p>
<label for="export-format">格式</label><select id="export-format" v-model="format"><option value="html">HTML</option><option value="pdf">PDF</option><option value="docx">DOCX</option></select>
<label>纸张 <select v-model="page"><option>A4</option><option>Letter</option></select></label>
<label><input v-model="title" type="checkbox">包含标题</label>
<p v-if="format !== 'html'">PDF / DOCX 使用浅色打印样式</p>
<button class="button-primary" :disabled="preparing || !editor.content.trim()" @click="start">{{ preparing ? '准备图表' : '开始导出' }}</button>
<button v-if="preparing" @click="controller?.abort()">取消准备</button>
<p v-if="error" role="alert">{{ error }}</p>
<ul><li v-for="job in jobs" :key="job.id">
<strong>{{ job.fileName ?? job.id }}</strong> · {{ labels[job.status] }}
<button v-if="job.status === 'completed'" @click="action(job, true)">下载</button>
<button v-if="['queued','running'].includes(job.status)" @click="action(job)">取消</button>
<p v-if="job.error" role="alert">{{ job.error }}</p>
<ul v-if="job.warnings.length" class="export-warnings" aria-label="导出警告"><li v-for="warning in job.warnings" :key="warning">{{ warning }}</li></ul>
</li></ul>
<button class="button-secondary" @click="emit('close')">关闭</button>
</section></AppDialog>
</template>
<style scoped>
.export-modal { width: min(640px, 100%); padding: 24px; background: var(--color-surface-primary); border: 1px solid var(--color-border-default); border-radius: 12px; }
label { display: inline-flex; align-items:center; gap: 8px; margin: 8px; } li { margin-block: 12px; overflow-wrap: anywhere; } button { margin: 6px; } .export-warnings { color: var(--color-warning); } [role=alert] { color:var(--color-error); }
</style>
@@ -156,20 +156,20 @@ function foldHeadings(action: 'toggle' | 'all' | 'none') {
}
})
}
const diagramPreviews = new Map<string, { source: string; apply: (value: HTMLElement) => void }>()
function renderDiagram(source: string, apply: (value: HTMLElement) => void) {
const diagramPreviews = new Map<string, { source: string; kind: string; apply: (value: HTMLElement) => void }>()
function renderDiagram(source: string, apply: (value: HTMLElement) => void, kind = 'mermaid') {
for (const [id, entry] of diagramPreviews) {
if (entry.apply === apply) diagramPreviews.delete(id)
}
const element = createMermaidPreview(source, themeStore.isDark, apply)
diagramPreviews.set(element.dataset.previewId!, { source, apply })
const element = createMermaidPreview(source, themeStore.isDark, apply, kind, themeStore.currentThemeId)
diagramPreviews.set(element.dataset.previewId!, { source, apply, kind })
return element
}
watch(() => themeStore.currentThemeId, () => {
const current = [...diagramPreviews.entries()]
diagramPreviews.clear()
for (const [id, entry] of current) {
if (editorRoot.value?.querySelector(`[id="${id}"]`)) entry.apply(renderDiagram(entry.source, entry.apply))
if (editorRoot.value?.querySelector(`[id="${id}"]`)) entry.apply(renderDiagram(entry.source, entry.apply, entry.kind))
}
}, { flush: 'post' })
@@ -333,8 +333,8 @@ onMounted(async () => {
...config,
languages: shikiLanguages(themeStore.resolvedCodeBlockTheme),
renderLanguage: renderCodeLanguage,
renderPreview: (language, content, applyPreview) => language.trim().toLowerCase() === 'mermaid'
? markdownPreferences.diagrams ? renderDiagram(content, applyPreview) : null
renderPreview: (language, content, applyPreview) => ['mermaid', 'function-plot'].includes(language.trim().toLowerCase())
? markdownPreferences.diagrams ? renderDiagram(content, applyPreview, language.trim().toLowerCase()) : null
: config.renderPreview(language, content, applyPreview),
extensions: [basicSetup, keymap.of([indentWithTab]), shikiEditorTheme(themeStore.resolvedCodeBlockTheme),
indentUnit.of(' '.repeat(markdownPreferences.indent)), CodeEditorState.tabSize.of(markdownPreferences.indent),
@@ -1,17 +1,18 @@
import { renderFunctionPlot } from '@/services/functionPlotService'
import { nextTick } from 'vue'
import { renderMermaid } from '@/services/mermaidService'
import { t } from '@/i18n'
import { appendDiagramControls } from '@/utils/diagramControls'
let previewId = 0
export function createMermaidPreview(source: string, dark: boolean, applyPreview: (value: HTMLElement) => void): HTMLElement {
export function createMermaidPreview(source: string, dark: boolean, applyPreview: (value: HTMLElement) => void, kind = 'mermaid', themeId = 'light'): HTMLElement {
// Each revision owns its element, so a slow render cannot replace newer content.
// Milkdown sanitizes Element input to its inner HTML; retain the revision
// marker and controls inside an otherwise disposable envelope.
const envelope = document.createElement('div')
const container = document.createElement('div')
envelope.append(container)
container.className = 'editor-mermaid-preview'
container.className = 'editor-mermaid-preview' + (kind === 'function-plot' ? ' function-plot-preview' : '')
container.id = `editor-mermaid-preview-${++previewId}`
envelope.dataset.previewId = container.id
container.setAttribute('aria-live', 'polite')
@@ -27,8 +28,11 @@ export function createMermaidPreview(source: string, dark: boolean, applyPreview
applyPreview(envelope.cloneNode(true) as HTMLElement)
}
}
void renderMermaid(source, { theme: dark ? 'dark' : 'light' }).then(result => {
if (result.warnings.length) {
void (kind === 'function-plot' ? new Promise<void>(resolve => setTimeout(resolve, 180)) : Promise.resolve()).then<{ svg: string; warnings: string[] }>(() => {
if (kind === 'function-plot' && !document.getElementById(container.id)) throw new Error('stale preview')
return kind === 'function-plot' ? renderFunctionPlot(source, themeId) : renderMermaid(source, { theme: dark ? 'dark' : 'light' })
}).then(result => {
if (result.warnings.length && (kind === 'mermaid' || !result.svg)) {
container.classList.add('has-error')
container.textContent = `${t('图表语法有误,可点击编辑修改:', 'Diagram syntax error. Choose Edit to fix:')} ${result.warnings.join('\n')}`
void publish()
@@ -37,6 +41,7 @@ export function createMermaidPreview(source: string, dark: boolean, applyPreview
// Mermaid runs in strict mode; Milkdown sanitizes the preview before insertion.
container.innerHTML = result.svg
appendDiagramControls(container)
if (result.warnings.length) { const warning = document.createElement('p'); warning.textContent = result.warnings.join('\n'); warning.setAttribute('role', 'status'); container.append(warning) }
void publish()
}).catch(() => {
container.textContent = t('图表渲染失败,请点击编辑检查源码。', 'Unable to render diagram. Choose Edit to inspect the source.')
@@ -69,7 +69,7 @@ it('offers fenced-code aliases and retains the LaTeX selector', () => {
it('offers every bundled Shiki language and alias', () => {
const languages = shikiLanguages('github-light')
expect(languages).toHaveLength(bundledLanguagesInfo.length + 1)
expect(languages).toHaveLength(bundledLanguagesInfo.length + 2)
for (const info of bundledLanguagesInfo) {
const language = languages.find(item => item.alias.includes(info.id))!
expect(language, info.id).toBeDefined()
@@ -60,6 +60,7 @@ export async function shikiLanguage(language: string, theme: CodeTheme): Promise
export function shikiLanguages(theme: CodeTheme): LanguageDescription[] {
return [
LanguageDescription.of({ name: 'function-plot', alias: ['Function Plot'], load: () => shikiLanguage('text', theme) }),
...bundledLanguagesInfo.map(info => LanguageDescription.of({
name: info.id,
alias: [info.name, ...(info.aliases ?? [])],