- 检索调优参数(rrf_k/rerank/rerank_candidates/score_threshold)透传到引擎实际执行 - Recall 去重,避免同一 Note 多 Block 重复导致 Recall 超 1 - RAG 运行改为后台异步执行:创建即 queued + 202,支持取消与 SSE 实时事件 - 数据集元数据校验,坏文件隔离跳过;citation_required 语义修正 - modes 空/重复校验;配置快照记录模型版本与索引元信息 Co-Authored-By: Claude Code <noreply@anthropic.com>
800 lines
20 KiB
Python
800 lines
20 KiB
Python
from datetime import datetime
|
||
from enum import Enum
|
||
from typing import Any, Literal
|
||
|
||
from pydantic import BaseModel, ConfigDict, Field, SecretStr, field_validator
|
||
|
||
|
||
class Contract(BaseModel):
|
||
model_config = ConfigDict(extra="forbid")
|
||
|
||
|
||
class PageMeta(Contract):
|
||
total: int = 0
|
||
limit: int = 50
|
||
offset: int = 0
|
||
|
||
|
||
class ErrorDetail(Contract):
|
||
code: str
|
||
message: str
|
||
details: dict[str, Any] = Field(default_factory=dict)
|
||
|
||
|
||
class ErrorResponse(Contract):
|
||
error: ErrorDetail
|
||
|
||
|
||
class OperationResponse(Contract):
|
||
status: Literal["accepted", "completed"]
|
||
resource_id: str | None = None
|
||
message: str | None = None
|
||
|
||
|
||
# Workspace boundary (single configured Vault in Web development mode)
|
||
class WorkspaceInfo(Contract):
|
||
vault_id: str = "default"
|
||
name: str
|
||
path: str
|
||
file_count: int = 0
|
||
indexed_note_count: int = 0
|
||
requires_refresh: bool = False
|
||
|
||
|
||
class WorkspaceEntry(Contract):
|
||
entry_id: str
|
||
name: str
|
||
path: str
|
||
type: Literal["file", "folder"]
|
||
note_id: str | None = None
|
||
children: list["WorkspaceEntry"] = Field(default_factory=list)
|
||
|
||
|
||
class WorkspaceSnapshot(Contract):
|
||
workspace: WorkspaceInfo
|
||
items: list[WorkspaceEntry] = Field(default_factory=list)
|
||
|
||
|
||
class WorkspaceOpenRequest(Contract):
|
||
path: str | None = None
|
||
|
||
|
||
class FolderCreateRequest(Contract):
|
||
parent: str = ""
|
||
name: str = Field(min_length=1)
|
||
|
||
|
||
class FolderRenameRequest(Contract):
|
||
path: str
|
||
new_name: str = Field(min_length=1)
|
||
|
||
|
||
class FolderDeleteRequest(Contract):
|
||
path: str
|
||
|
||
|
||
# Notes and retrieval
|
||
class NoteBlock(Contract):
|
||
block_id: str
|
||
note_id: str
|
||
heading_path: list[str] = Field(default_factory=list)
|
||
start_offset: int
|
||
end_offset: int
|
||
content: str
|
||
content_hash: str
|
||
token_count: int
|
||
|
||
|
||
class NoteSummary(Contract):
|
||
note_id: str
|
||
title: str
|
||
file_path: str
|
||
tags: list[str] = Field(default_factory=list)
|
||
created_at: datetime
|
||
updated_at: datetime
|
||
|
||
|
||
class Note(NoteSummary):
|
||
markdown: str
|
||
blocks: list[NoteBlock] = Field(default_factory=list)
|
||
|
||
|
||
class NoteListResponse(Contract):
|
||
items: list[NoteSummary] = Field(default_factory=list)
|
||
page: PageMeta = Field(default_factory=PageMeta)
|
||
|
||
|
||
class NoteCreateRequest(Contract):
|
||
title: str = Field(min_length=1)
|
||
markdown: str = ""
|
||
folder: str | None = None
|
||
tags: list[str] = Field(default_factory=list)
|
||
|
||
|
||
class NoteUpdateRequest(Contract):
|
||
title: str | None = None
|
||
markdown: str | None = None
|
||
tags: list[str] | None = None
|
||
|
||
|
||
class NoteMoveRequest(Contract):
|
||
folder: str
|
||
|
||
|
||
class NoteRenameRequest(Contract):
|
||
file_name: str = Field(min_length=1)
|
||
|
||
|
||
class SearchMode(str, Enum):
|
||
fts = "fts"
|
||
vector = "vector"
|
||
hybrid = "hybrid"
|
||
|
||
|
||
class SearchRequest(Contract):
|
||
query: str = Field(min_length=1)
|
||
mode: SearchMode = SearchMode.hybrid
|
||
folders: list[str] = Field(default_factory=list)
|
||
note_ids: list[str] = Field(default_factory=list)
|
||
tags: list[str] = Field(default_factory=list)
|
||
created_from: datetime | None = None
|
||
created_to: datetime | None = None
|
||
updated_from: datetime | None = None
|
||
updated_to: datetime | None = None
|
||
limit: int = Field(default=20, ge=1, le=100)
|
||
offset: int = Field(default=0, ge=0)
|
||
include_snippet: bool = True
|
||
# 检索调优参数(Benchmark 与 Skill 共用):控制 RRF / 精排 / 候选池 / 分数阈值。
|
||
# rerank_candidates=None 表示对全部候选精排(保留原有行为),Benchmark 传显式值。
|
||
rrf_k: int = Field(default=60, ge=1)
|
||
rerank: bool = True
|
||
rerank_candidates: int | None = Field(default=None, ge=1)
|
||
score_threshold: float = Field(default=0.0, ge=0.0)
|
||
|
||
|
||
class Citation(Contract):
|
||
citation_id: str
|
||
note_id: str
|
||
block_id: str
|
||
file_path: str
|
||
heading_path: list[str] = Field(default_factory=list)
|
||
start_offset: int | None = None
|
||
end_offset: int | None = None
|
||
source_audio: str | None = None
|
||
start_time: float | None = None
|
||
end_time: float | None = None
|
||
speaker: str | None = None
|
||
|
||
|
||
class SearchResult(Contract):
|
||
note_id: str
|
||
block_id: str
|
||
title: str
|
||
file_path: str
|
||
heading_path: list[str] = Field(default_factory=list)
|
||
snippet: str | None = None
|
||
score: float
|
||
citation: Citation
|
||
|
||
|
||
class SearchResponse(Contract):
|
||
query: str
|
||
mode: SearchMode
|
||
items: list[SearchResult] = Field(default_factory=list)
|
||
page: PageMeta = Field(default_factory=PageMeta)
|
||
|
||
|
||
# Model, chat and tools
|
||
class MessageRole(str, Enum):
|
||
system = "system"
|
||
user = "user"
|
||
assistant = "assistant"
|
||
tool = "tool"
|
||
|
||
|
||
class Message(Contract):
|
||
role: MessageRole
|
||
content: str
|
||
name: str | None = None
|
||
tool_call_id: str | None = None
|
||
tool_calls: list["ToolCall"] = Field(default_factory=list)
|
||
|
||
|
||
class ToolDefinition(Contract):
|
||
name: str
|
||
description: str
|
||
parameters: dict[str, Any] = Field(default_factory=dict)
|
||
permission: str | None = None
|
||
source: Literal["builtin", "plugin"] = "builtin"
|
||
|
||
|
||
class ToolCall(Contract):
|
||
tool_call_id: str
|
||
name: str
|
||
arguments: dict[str, Any] = Field(default_factory=dict)
|
||
|
||
|
||
class ToolResult(Contract):
|
||
tool_call_id: str
|
||
name: str
|
||
success: bool
|
||
output: Any | None = None
|
||
error_code: str | None = None
|
||
error_message: str | None = None
|
||
duration_ms: int | None = None
|
||
|
||
|
||
class ToolListResponse(Contract):
|
||
items: list[ToolDefinition] = Field(default_factory=list)
|
||
|
||
|
||
class ModelCapability(str, Enum):
|
||
chat = "chat"
|
||
vision = "vision"
|
||
tool_calling = "tool_calling"
|
||
reasoning = "reasoning"
|
||
streaming = "streaming"
|
||
structured_output = "structured_output"
|
||
embedding = "embedding"
|
||
|
||
|
||
class ModelRequest(Contract):
|
||
provider_id: str
|
||
model: str
|
||
system: str | None = None
|
||
messages: list[Message]
|
||
tools: list[ToolDefinition] = Field(default_factory=list)
|
||
temperature: float | None = Field(default=None, ge=0, le=2)
|
||
max_tokens: int | None = Field(default=None, ge=1)
|
||
response_format: dict[str, Any] | None = None
|
||
attachments: list[str] = Field(default_factory=list)
|
||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||
|
||
|
||
class ChatRequest(ModelRequest):
|
||
conversation_id: str | None = None
|
||
use_rag: bool = True
|
||
retrieval: SearchRequest | None = None
|
||
|
||
|
||
class ModelEventType(str, Enum):
|
||
text_delta = "TextDelta"
|
||
thinking_delta = "ThinkingDelta"
|
||
tool_call_start = "ToolCallStart"
|
||
tool_call_delta = "ToolCallDelta"
|
||
tool_call_end = "ToolCallEnd"
|
||
usage = "Usage"
|
||
error = "Error"
|
||
done = "Done"
|
||
|
||
|
||
class ModelEvent(Contract):
|
||
event: ModelEventType
|
||
sequence: int = 0
|
||
data: dict[str, Any] = Field(default_factory=dict)
|
||
timestamp: datetime
|
||
|
||
|
||
# Agent
|
||
class AgentRunStatus(str, Enum):
|
||
queued = "queued"
|
||
running = "running"
|
||
waiting_permission = "waiting_permission"
|
||
completed = "completed"
|
||
failed = "failed"
|
||
cancelled = "cancelled"
|
||
|
||
|
||
class AgentRunCreateRequest(Contract):
|
||
input: str = Field(min_length=1)
|
||
provider_id: str
|
||
model: str
|
||
skill_id: str | None = None
|
||
allowed_tools: list[str] = Field(default_factory=list)
|
||
max_steps: int = Field(default=10, ge=1, le=100)
|
||
tool_timeout_seconds: int = Field(default=30, ge=1)
|
||
run_timeout_seconds: int = Field(default=300, ge=1)
|
||
token_budget: int | None = Field(default=None, ge=1)
|
||
max_concurrent_tools: int = Field(default=1, ge=1)
|
||
allow_network: bool = False
|
||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||
|
||
|
||
class AgentRun(Contract):
|
||
run_id: str
|
||
status: AgentRunStatus
|
||
input: str
|
||
provider_id: str
|
||
model: str
|
||
skill_id: str | None = None
|
||
current_step: int = 0
|
||
max_steps: int
|
||
token_budget: int | None = None
|
||
cancelled: bool = False
|
||
output: str | None = None
|
||
error_code: str | None = None
|
||
error_message: str | None = None
|
||
token_usage: int = 0
|
||
tool_results: list[ToolResult] = Field(default_factory=list)
|
||
citations: list[Citation] = Field(default_factory=list)
|
||
created_at: datetime
|
||
updated_at: datetime
|
||
|
||
|
||
class AgentRunListResponse(Contract):
|
||
items: list[AgentRun] = Field(default_factory=list)
|
||
page: PageMeta = Field(default_factory=PageMeta)
|
||
|
||
|
||
class AgentEventType(str, Enum):
|
||
run_started = "RunStarted"
|
||
text_delta = "TextDelta"
|
||
thinking_delta = "ThinkingDelta"
|
||
tool_call = "ToolCall"
|
||
tool_result = "ToolResult"
|
||
permission_required = "PermissionRequired"
|
||
usage = "Usage"
|
||
citation = "Citation"
|
||
model_call_started = "ModelCallStarted"
|
||
model_call_completed = "ModelCallCompleted"
|
||
model_call_failed = "ModelCallFailed"
|
||
permission_resolved = "PermissionResolved"
|
||
run_completed = "RunCompleted"
|
||
run_failed = "RunFailed"
|
||
run_cancelled = "RunCancelled"
|
||
|
||
|
||
class AgentEvent(Contract):
|
||
event: AgentEventType
|
||
run_id: str
|
||
sequence: int
|
||
data: dict[str, Any] = Field(default_factory=dict)
|
||
timestamp: datetime
|
||
|
||
|
||
class AgentTraceSummary(Contract):
|
||
model_calls: int = 0
|
||
tool_calls: int = 0
|
||
duration_ms: int = 0
|
||
token_usage: int = 0
|
||
errors: int = 0
|
||
|
||
|
||
class AgentTraceResponse(Contract):
|
||
run_id: str
|
||
status: AgentRunStatus
|
||
items: list[AgentEvent] = Field(default_factory=list)
|
||
next_sequence: int
|
||
has_more: bool = False
|
||
summary: AgentTraceSummary = Field(default_factory=AgentTraceSummary)
|
||
config_snapshot: dict[str, Any] = Field(default_factory=dict)
|
||
|
||
|
||
class PermissionDecisionRequest(Contract):
|
||
decision: Literal["allow_once", "allow_session", "deny"]
|
||
|
||
|
||
# Skills and plugins
|
||
class RetrievalConfig(Contract):
|
||
top_k: int = Field(default=10, ge=1, le=100)
|
||
rerank: bool = True
|
||
citation: bool = True
|
||
|
||
|
||
class SkillModelConfig(Contract):
|
||
required_capabilities: list[ModelCapability] = Field(default_factory=list)
|
||
|
||
|
||
class SkillManifest(Contract):
|
||
skill_id: str
|
||
name: str
|
||
version: str
|
||
description: str = ""
|
||
permissions: list[str] = Field(default_factory=list)
|
||
tools: list[str] = Field(default_factory=list)
|
||
retrieval: RetrievalConfig = Field(default_factory=RetrievalConfig)
|
||
model: SkillModelConfig = Field(default_factory=SkillModelConfig)
|
||
|
||
|
||
class SkillStatus(str, Enum):
|
||
installed = "installed"
|
||
disabled = "disabled"
|
||
ready = "ready"
|
||
dependency_missing = "dependency_missing"
|
||
permission_required = "permission_required"
|
||
error = "error"
|
||
|
||
|
||
class Skill(Contract):
|
||
manifest: SkillManifest
|
||
status: SkillStatus
|
||
enabled: bool = False
|
||
missing_dependencies: list[str] = Field(default_factory=list)
|
||
|
||
|
||
class SkillListResponse(Contract):
|
||
items: list[Skill] = Field(default_factory=list)
|
||
|
||
|
||
class ExtensionInstallRequest(Contract):
|
||
package_path: str
|
||
|
||
|
||
class PluginBackend(Contract):
|
||
type: Literal["mcp", "internal_rpc", "none"] = "none"
|
||
transport: Literal["stdio", "http", "none"] = "none"
|
||
command: str | None = None
|
||
args: list[str] = Field(default_factory=list)
|
||
startup_timeout_seconds: int = Field(default=10, ge=1, le=60)
|
||
tool_timeout_seconds: int = Field(default=30, ge=1, le=600)
|
||
|
||
|
||
class PluginContribution(Contract):
|
||
tools: list[str] = Field(default_factory=list)
|
||
commands: list[str] = Field(default_factory=list)
|
||
importers: list[str] = Field(default_factory=list)
|
||
exporters: list[str] = Field(default_factory=list)
|
||
panels: list[str] = Field(default_factory=list)
|
||
settings_sections: list[str] = Field(default_factory=list)
|
||
|
||
|
||
class PluginManifest(Contract):
|
||
plugin_id: str
|
||
name: str
|
||
version: str
|
||
description: str = ""
|
||
permissions: list[str] = Field(default_factory=list)
|
||
contributes: PluginContribution = Field(default_factory=PluginContribution)
|
||
backend: PluginBackend = Field(default_factory=PluginBackend)
|
||
|
||
|
||
class PluginStatus(str, Enum):
|
||
installed = "installed"
|
||
disabled = "disabled"
|
||
starting = "starting"
|
||
ready = "ready"
|
||
error = "error"
|
||
dependency_missing = "dependency_missing"
|
||
permission_required = "permission_required"
|
||
|
||
|
||
class Plugin(Contract):
|
||
manifest: PluginManifest
|
||
status: PluginStatus
|
||
enabled: bool = False
|
||
granted_permissions: list[str] = Field(default_factory=list)
|
||
error_message: str | None = None
|
||
|
||
|
||
class PluginListResponse(Contract):
|
||
items: list[Plugin] = Field(default_factory=list)
|
||
|
||
|
||
class PluginHostState(str, Enum):
|
||
stopped = "stopped"
|
||
starting = "starting"
|
||
ready = "ready"
|
||
unhealthy = "unhealthy"
|
||
error = "error"
|
||
|
||
|
||
class PluginHostStatus(Contract):
|
||
plugin_id: str
|
||
backend_type: Literal["mcp", "internal_rpc", "none"]
|
||
transport: Literal["stdio", "http", "none"]
|
||
status: PluginHostState
|
||
tools_count: int = 0
|
||
started_at: datetime | None = None
|
||
last_seen_at: datetime | None = None
|
||
protocol_version: str | None = None
|
||
server_name: str | None = None
|
||
server_version: str | None = None
|
||
error: str | None = None
|
||
|
||
|
||
class PluginPermissionGrantRequest(Contract):
|
||
permissions: list[str] = Field(default_factory=list)
|
||
|
||
|
||
# Providers
|
||
class ProviderType(str, Enum):
|
||
mock = "mock"
|
||
openai_responses = "openai_responses"
|
||
openai_chat = "openai_chat"
|
||
openai_compatible = "openai_compatible"
|
||
anthropic_messages = "anthropic_messages"
|
||
ollama = "ollama"
|
||
|
||
|
||
class ProviderConfig(Contract):
|
||
provider_id: str
|
||
provider_type: ProviderType
|
||
name: str
|
||
base_url: str | None = None
|
||
default_model: str | None = None
|
||
credential_id: str | None = None
|
||
enabled: bool = True
|
||
capabilities: list[ModelCapability] = Field(default_factory=list)
|
||
|
||
|
||
class ProviderCreateRequest(Contract):
|
||
provider_type: ProviderType
|
||
name: str
|
||
base_url: str | None = None
|
||
default_model: str | None = None
|
||
credential_id: str | None = None
|
||
enabled: bool = True
|
||
|
||
|
||
class ProviderUpdateRequest(Contract):
|
||
name: str | None = None
|
||
base_url: str | None = None
|
||
default_model: str | None = None
|
||
credential_id: str | None = None
|
||
enabled: bool | None = None
|
||
|
||
|
||
class ProviderListResponse(Contract):
|
||
items: list[ProviderConfig] = Field(default_factory=list)
|
||
|
||
|
||
class ProviderPreset(Contract):
|
||
preset_id: str
|
||
name: str
|
||
provider_type: ProviderType
|
||
base_url: str
|
||
default_credential_id: str | None = None
|
||
requires_credential: bool = True
|
||
|
||
|
||
class ProviderPresetListResponse(Contract):
|
||
items: list[ProviderPreset] = Field(default_factory=list)
|
||
|
||
|
||
class CredentialWriteRequest(Contract):
|
||
api_key: SecretStr = Field(min_length=1, max_length=8192)
|
||
|
||
|
||
class CredentialStatus(Contract):
|
||
credential_id: str
|
||
configured: bool
|
||
|
||
|
||
class ModelInfo(Contract):
|
||
model: str
|
||
display_name: str
|
||
capabilities: list[ModelCapability] = Field(default_factory=list)
|
||
|
||
|
||
class ProviderModelsResponse(Contract):
|
||
provider_id: str
|
||
items: list[ModelInfo] = Field(default_factory=list)
|
||
|
||
|
||
class ProviderTestRequest(Contract):
|
||
provider_id: str
|
||
model: str | None = None
|
||
credential_context_id: str | None = None
|
||
|
||
|
||
class ProviderTestResponse(Contract):
|
||
provider_id: str
|
||
success: bool
|
||
latency_ms: int | None = None
|
||
message: str
|
||
|
||
|
||
# Tasks, media and index
|
||
class TaskStatus(str, Enum):
|
||
todo = "todo"
|
||
in_progress = "in_progress"
|
||
done = "done"
|
||
cancelled = "cancelled"
|
||
|
||
|
||
class Task(Contract):
|
||
task_id: str
|
||
title: str
|
||
description: str = ""
|
||
status: TaskStatus = TaskStatus.todo
|
||
note_id: str | None = None
|
||
due_at: datetime | None = None
|
||
created_at: datetime
|
||
updated_at: datetime
|
||
|
||
|
||
class TaskCreateRequest(Contract):
|
||
title: str = Field(min_length=1)
|
||
description: str = ""
|
||
note_id: str | None = None
|
||
due_at: datetime | None = None
|
||
|
||
|
||
class TaskUpdateRequest(Contract):
|
||
title: str | None = None
|
||
description: str | None = None
|
||
status: TaskStatus | None = None
|
||
note_id: str | None = None
|
||
due_at: datetime | None = None
|
||
|
||
|
||
class TaskListResponse(Contract):
|
||
items: list[Task] = Field(default_factory=list)
|
||
page: PageMeta = Field(default_factory=PageMeta)
|
||
|
||
|
||
class TranscriptionRequest(Contract):
|
||
attachment_id: str
|
||
language: str | None = None
|
||
diarization: bool = False
|
||
|
||
|
||
class TranscriptionJob(Contract):
|
||
job_id: str
|
||
attachment_id: str
|
||
status: Literal["queued", "processing", "completed", "failed"]
|
||
text: str | None = None
|
||
error_code: str | None = None
|
||
error_message: str | None = None
|
||
created_at: datetime
|
||
|
||
|
||
class IndexStatus(Contract):
|
||
status: Literal["idle", "queued", "running", "failed"] = "idle"
|
||
pending_jobs: int = 0
|
||
active_job_id: str | None = None
|
||
last_completed_at: datetime | None = None
|
||
error_message: str | None = None
|
||
|
||
|
||
class IndexRebuildRequest(Contract):
|
||
scope: Literal["all", "notes", "vectors"] = "all"
|
||
note_ids: list[str] = Field(default_factory=list)
|
||
force: bool = False
|
||
|
||
|
||
class IndexJob(Contract):
|
||
job_id: str
|
||
status: Literal["queued", "running", "completed", "failed"]
|
||
scope: Literal["all", "notes", "vectors"]
|
||
created_at: datetime
|
||
|
||
|
||
# Benchmark
|
||
class BenchmarkKind(str, Enum):
|
||
rag = "rag"
|
||
agent = "agent"
|
||
|
||
|
||
class BenchmarkStatus(str, Enum):
|
||
queued = "queued"
|
||
running = "running"
|
||
completed = "completed"
|
||
failed = "failed"
|
||
cancelled = "cancelled"
|
||
|
||
|
||
class RAGDatasetCase(Contract):
|
||
case_id: str
|
||
query: str = Field(min_length=1)
|
||
expected_note_ids: list[str] = Field(default_factory=list)
|
||
expected_block_ids: list[str] = Field(default_factory=list)
|
||
citation_required: bool = False
|
||
tags: list[str] = Field(default_factory=list)
|
||
|
||
|
||
class RAGRetrievalConfig(Contract):
|
||
"""RAG Benchmark 的检索参数。top_k 映射到 SearchRequest.limit,
|
||
其余参数透传到 SearchRequest,由检索引擎实际执行。"""
|
||
|
||
top_k: int = Field(default=10, ge=1, le=100)
|
||
rrf_k: int = Field(default=60, ge=1)
|
||
rerank: bool = True
|
||
rerank_candidates: int = Field(default=20, ge=1)
|
||
score_threshold: float = Field(default=0.0, ge=0.0)
|
||
|
||
|
||
class RAGRunRequest(Contract):
|
||
dataset_id: str = Field(min_length=1)
|
||
modes: list[SearchMode] = Field(
|
||
default_factory=lambda: [SearchMode.fts, SearchMode.vector, SearchMode.hybrid],
|
||
min_length=1,
|
||
)
|
||
retrieval: RAGRetrievalConfig = Field(default_factory=RAGRetrievalConfig)
|
||
repeat: int = Field(default=1, ge=1, le=10)
|
||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||
|
||
@field_validator("modes")
|
||
@classmethod
|
||
def _no_duplicate_modes(cls, value: list[SearchMode]) -> list[SearchMode]:
|
||
if len(value) != len(set(value)):
|
||
raise ValueError("modes must not contain duplicates")
|
||
return value
|
||
|
||
|
||
class RAGMetrics(Contract):
|
||
hit_at_1: float = 0.0
|
||
hit_at_5: float = 0.0
|
||
recall_at_k: float = 0.0
|
||
mrr: float = 0.0
|
||
citation_hit_rate: float = 0.0
|
||
p50_latency_ms: float = 0.0
|
||
p95_latency_ms: float = 0.0
|
||
|
||
|
||
class BenchmarkDatasetInfo(Contract):
|
||
dataset_id: str
|
||
kind: BenchmarkKind
|
||
version: str
|
||
description: str = ""
|
||
case_count: int
|
||
content_hash: str
|
||
|
||
|
||
class BenchmarkDatasetListResponse(Contract):
|
||
items: list[BenchmarkDatasetInfo] = Field(default_factory=list)
|
||
|
||
|
||
class BenchmarkRun(Contract):
|
||
run_id: str
|
||
kind: BenchmarkKind
|
||
dataset_id: str
|
||
dataset_hash: str
|
||
status: BenchmarkStatus
|
||
progress: float | None = None
|
||
metrics: dict[str, Any] | None = None
|
||
config_snapshot: dict[str, Any] = Field(default_factory=dict)
|
||
error: str | None = None
|
||
created_at: datetime
|
||
started_at: datetime | None = None
|
||
completed_at: datetime | None = None
|
||
|
||
|
||
class BenchmarkRunListResponse(Contract):
|
||
items: list[BenchmarkRun] = Field(default_factory=list)
|
||
page: PageMeta = Field(default_factory=PageMeta)
|
||
|
||
|
||
class BenchmarkEventType(str, Enum):
|
||
run_started = "RunStarted"
|
||
case_completed = "CaseCompleted"
|
||
run_completed = "RunCompleted"
|
||
run_failed = "RunFailed"
|
||
|
||
|
||
class BenchmarkEvent(Contract):
|
||
event: BenchmarkEventType
|
||
run_id: str
|
||
sequence: int
|
||
data: dict[str, Any] = Field(default_factory=dict)
|
||
timestamp: datetime
|
||
|
||
|
||
class RAGCaseResult(Contract):
|
||
case_id: str
|
||
mode: SearchMode
|
||
repeat: int
|
||
latency_ms: float
|
||
retrieved_note_ids: list[str] = Field(default_factory=list)
|
||
retrieved_block_ids: list[str] = Field(default_factory=list)
|
||
hit_at_1: bool = False
|
||
hit_at_5: bool = False
|
||
recall: float = 0.0
|
||
reciprocal_rank: float = 0.0
|
||
citation_hit: bool = False
|
||
# 该 Case 是否声明了 expected_block_ids(决定是否计入 citation_hit_rate 分母)
|
||
citation_applicable: bool = False
|
||
error: str | None = None
|
||
|
||
|
||
class BenchmarkReport(Contract):
|
||
run_id: str
|
||
kind: BenchmarkKind
|
||
dataset_id: str
|
||
dataset_hash: str
|
||
status: BenchmarkStatus
|
||
config_snapshot: dict[str, Any] = Field(default_factory=dict)
|
||
metrics: dict[str, Any] = Field(default_factory=dict)
|
||
cases: list[RAGCaseResult] = Field(default_factory=list)
|
||
error: str | None = None
|