feat(provider): 完成阶段E协议适配、国内预设与模型路由

This commit is contained in:
2026-09-04 06:19:32 +08:00
parent 9b8b10cdb1
commit 1fe75e3fd2
57 changed files with 4208 additions and 592 deletions
+163
View File
@@ -0,0 +1,163 @@
"""Native Anthropic Messages protocol with incrementally decoded content blocks."""
import json
from contextlib import aclosing
from app.contracts import MessageRole, ModelEventType, ModelRequest
from app.providers.base import ProviderError, ProviderToolCall, ProviderTurn
from app.providers.http_base import (
UsageTracker, check_error, decode_tool_arguments, invalid_response, list_value,
object_value, string_value, token_count, truncated_stream,
)
from app.providers.openai_compatible import OpenAICompatibleProvider
from app.providers.tool_names import mapped_tool_names
class AnthropicMessagesProvider(OpenAICompatibleProvider):
stream_path = "/messages"
def _headers(self) -> dict[str, str]:
headers = super()._headers()
authorization = headers.pop("Authorization", None)
if authorization:
headers["x-api-key"] = authorization.removeprefix("Bearer ")
headers["anthropic-version"] = "2023-06-01"
return headers
def _payload(self, request: ModelRequest, *, stream: bool) -> dict[str, object]:
systems = [request.system] if request.system else []
messages = []
for message in request.messages:
if message.role == MessageRole.system:
systems.append(message.content)
continue
if message.role == MessageRole.tool:
if not message.tool_call_id:
raise ProviderError("PROVIDER_INVALID_REQUEST", "Tool result requires a call identifier.")
role = "user"
content = [{"type": "tool_result", "tool_use_id": message.tool_call_id, "content": message.content}]
else:
role = message.role.value
content = [{"type": "text", "text": message.content}] if message.content else []
content += [{"type": "tool_use", "id": call.tool_call_id, "name": call.name,
"input": call.arguments} for call in message.tool_calls]
if not content:
continue
if messages and messages[-1]["role"] == role:
messages[-1]["content"].extend(content)
else:
messages.append({"role": role, "content": content})
payload: dict[str, object] = {"model": request.model, "messages": messages,
"max_tokens": request.max_tokens or 4096, "stream": stream}
if systems:
payload["system"] = "\n\n".join(systems)
if request.tools:
payload["tools"] = [{"name": tool.name, "description": tool.description,
"input_schema": tool.parameters} for tool in request.tools]
if request.temperature is not None:
payload["temperature"] = request.temperature
if request.response_format is not None:
format_ = request.response_format
if format_.get("type") != "json_schema":
raise ProviderError("PROVIDER_INVALID_REQUEST", "Messages requires a JSON schema response format.")
schema = object_value(format_.get("json_schema"))
payload["output_config"] = {"format": {"type": "json_schema", "schema": object_value(schema.get("schema"))}}
return payload
@mapped_tool_names
async def complete(self, request: ModelRequest) -> ProviderTurn:
data = await self._request("POST", self.stream_path, json=self._payload(request, stream=False))
texts = []
calls = []
for raw in list_value(data.get("content")):
block = object_value(raw)
if block.get("type") == "text":
texts.append(string_value(block.get("text")))
elif block.get("type") == "tool_use":
calls.append(ProviderToolCall(
tool_call_id=string_value(block.get("id"), nonempty=True),
name=string_value(block.get("name"), nonempty=True),
arguments=decode_tool_arguments(block.get("input")),
))
return ProviderTurn(text="".join(texts) or None, tool_calls=calls,
**UsageTracker(cache_tokens=True).update(data.get("usage") or {}))
async def _events(self, request: ModelRequest):
blocks: dict[int, dict] = {}
usage = UsageTracker(cache_tokens=True)
started = False
async with aclosing(self._stream_json(self._payload(request, stream=True))) as chunks:
async for data in chunks:
kind = string_value(data.get("type"), nonempty=True)
if kind == "message_start":
if started:
raise invalid_response()
started = True
message = object_value(data.get("message"))
check_error(message)
if message.get("usage") is not None:
yield ModelEventType.usage, usage.update(message["usage"])
elif kind == "content_block_start":
index = token_count(data.get("index"))
if not started or index in blocks:
raise invalid_response()
block = dict(object_value(data.get("content_block")))
blocks[index] = block
block["closed"] = False
if block.get("type") == "tool_use":
block["id"] = string_value(block.get("id"), nonempty=True)
block["name"] = string_value(block.get("name"), nonempty=True)
block["arguments"] = ""
block["input"] = object_value(block.get("input", {}))
yield ModelEventType.tool_call_start, {"tool_call_id": block["id"], "name": block["name"]}
elif block.get("type") == "text" and block.get("text"):
yield ModelEventType.text_delta, {"text": string_value(block["text"])}
elif block.get("type") == "thinking" and block.get("thinking"):
yield ModelEventType.thinking_delta, {"text": string_value(block["thinking"])}
elif kind == "content_block_delta":
block = blocks.get(token_count(data.get("index")))
if block is None or block["closed"]:
raise invalid_response()
delta = object_value(data.get("delta"))
delta_type = delta.get("type")
if delta_type == "text_delta":
if block.get("type") != "text":
raise invalid_response()
yield ModelEventType.text_delta, {"text": string_value(delta.get("text"))}
elif delta_type == "thinking_delta":
if block.get("type") != "thinking":
raise invalid_response()
yield ModelEventType.thinking_delta, {"text": string_value(delta.get("thinking"))}
elif delta_type == "input_json_delta" and block.get("type") == "tool_use":
fragment = string_value(delta.get("partial_json"))
block["arguments"] += fragment
yield ModelEventType.tool_call_delta, {"tool_call_id": block["id"], "arguments_delta": fragment}
# Signatures and future delta types have no representation in ModelEvent.
elif kind == "content_block_stop":
block = blocks.get(token_count(data.get("index")))
if block is None or block["closed"]:
raise invalid_response()
block["closed"] = True
if block.get("type") == "tool_use":
if block["arguments"]:
decode_tool_arguments(block["arguments"])
else:
yield ModelEventType.tool_call_delta, {
"tool_call_id": block["id"], "arguments_delta": json.dumps(block["input"]),
}
yield ModelEventType.tool_call_end, {"tool_call_id": block["id"]}
elif kind == "message_delta":
if not started:
raise invalid_response()
object_value(data.get("delta"))
if data.get("usage") is not None:
yield ModelEventType.usage, usage.update(data["usage"])
elif kind == "message_stop":
if not started:
raise invalid_response()
if any(not block["closed"] for block in blocks.values()):
raise truncated_stream()
return
elif kind == "[DONE]":
raise truncated_stream()
raise truncated_stream()
+46 -1
View File
@@ -16,6 +16,18 @@ class ProviderFactory:
self.credentials = ProviderCredentialResolver(credentials)
def build(self, config: ProviderConfig) -> ModelProvider:
if config.provider_type == ProviderType.openai_responses:
from app.providers.openai_responses import OpenAIResponsesProvider
return OpenAIResponsesProvider(
base_url=config.base_url or "https://api.openai.com/v1",
credential_id=config.credential_id, credentials=self.credentials,
)
if config.provider_type == ProviderType.anthropic_messages:
from app.providers.anthropic_messages import AnthropicMessagesProvider
return AnthropicMessagesProvider(
base_url=config.base_url or "https://api.anthropic.com/v1",
credential_id=config.credential_id, credentials=self.credentials,
)
if config.provider_type in {
ProviderType.openai_chat,
ProviderType.openai_compatible,
@@ -31,7 +43,7 @@ class ProviderFactory:
@staticmethod
def presets() -> list[ProviderPreset]:
return [
presets = [
ProviderPreset(
preset_id="openai",
name="OpenAI",
@@ -54,12 +66,45 @@ class ProviderFactory:
requires_credential=False,
),
]
# General API endpoints. Coding-plan endpoints and keys are separate products.
domestic = [
("kimi", "Kimi / 月之暗面", "https://api.moonshot.cn/v1", [], "长上下文对话;模型以账号权限为准。"),
("qwen", "阿里云百炼", "https://dashscope.aliyuncs.com/compatible-mode/v1", [ModelCapability.embedding], "中国内地兼容接口;海外地域需修改地址。"),
("zhipu", "智谱 GLM", "https://open.bigmodel.cn/api/paas/v4", [ModelCapability.embedding], "通用 APICoding Plan 请使用其专用地址。"),
("volcengine", "火山方舟 / 豆包", "https://ark.cn-beijing.volces.com/api/v3", [ModelCapability.embedding], "按账号填写模型 ID 或推理接入点 ID。"),
("siliconflow", "硅基流动", "https://api.siliconflow.cn/v1", [ModelCapability.embedding, ModelCapability.transcription], "支持兼容 Embedding 和音频转写接口。"),
("baidu", "百度千帆", "https://qianfan.baidubce.com/v2", [ModelCapability.embedding], "使用千帆 API Key;模型列表取决于账号。"),
("hunyuan", "腾讯混元", "https://api.hunyuan.cloud.tencent.com/v1", [], "OpenAI 兼容对话接口。"),
("minimax", "MiniMax", "https://api.minimaxi.com/v1", [], "文本对话兼容接口;其他媒体协议需独立适配。"),
("stepfun", "阶跃星辰", "https://api.stepfun.com/v1", [], "通用 API;Step Plan 请使用其专用地址。"),
]
for preset_id, name, url, extra, description in domestic:
presets.append(ProviderPreset(
preset_id=preset_id, name=name, provider_type=ProviderType.openai_compatible,
base_url=url, default_credential_id=preset_id, logo_id=preset_id,
capabilities=[ModelCapability.chat, *extra], description=description,
))
presets.extend([
ProviderPreset(preset_id="openai-responses", name="OpenAI Responses", provider_type=ProviderType.openai_responses,
base_url="https://api.openai.com/v1", default_credential_id="openai", logo_id="openai"),
ProviderPreset(preset_id="anthropic", name="Anthropic / Claude", provider_type=ProviderType.anthropic_messages,
base_url="https://api.anthropic.com/v1", default_credential_id="anthropic", logo_id="anthropic"),
])
for preset in presets:
if preset.logo_id == "custom":
preset.logo_id = preset.preset_id
if not preset.capabilities:
preset.capabilities = [ModelCapability.chat]
presets[0].capabilities += [ModelCapability.embedding, ModelCapability.transcription]
return presets
@staticmethod
def capabilities(provider_type: ProviderType) -> list[ModelCapability]:
if provider_type in {
ProviderType.openai_chat,
ProviderType.openai_compatible,
ProviderType.openai_responses,
ProviderType.anthropic_messages,
}:
return [
ModelCapability.chat,
+230
View File
@@ -1,9 +1,13 @@
import json
from collections.abc import AsyncIterator
from contextlib import aclosing
from datetime import datetime, timezone
import httpx
from app.contracts import ModelEvent, ModelEventType, ModelRequest
from app.providers.base import ProviderError, ProviderTurn
from app.providers.tool_names import prepare_tool_names
class TurnStreamingMixin:
@@ -80,3 +84,229 @@ def decode_tool_arguments(value: object) -> dict[str, object]:
if not isinstance(decoded, dict):
raise ProviderError("PROVIDER_INVALID_RESPONSE", "Tool arguments must be an object.")
return decoded
def invalid_response() -> ProviderError:
return ProviderError("PROVIDER_INVALID_RESPONSE", "Provider returned an invalid response.")
def truncated_stream() -> ProviderError:
return ProviderError("PROVIDER_STREAM_TRUNCATED", "Provider stream ended before completion.")
def object_value(value: object) -> dict:
if not isinstance(value, dict):
raise invalid_response()
return value
def list_value(value: object) -> list:
if not isinstance(value, list):
raise invalid_response()
return value
def string_value(value: object, *, nonempty: bool = False) -> str:
if not isinstance(value, str) or (nonempty and not value):
raise invalid_response()
return value
def token_count(value: object) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
raise invalid_response()
return value
def remote_error(value: object) -> ProviderError:
# Never reflect upstream messages, URLs, request bodies or credentials.
error = value if isinstance(value, dict) else {}
code = error.get("code") or error.get("type")
mapping = {
"authentication_error": "PROVIDER_AUTH_FAILED",
"invalid_api_key": "PROVIDER_AUTH_FAILED",
"permission_error": "PROVIDER_AUTH_FAILED",
"rate_limit_error": "PROVIDER_RATE_LIMITED",
"rate_limit_exceeded": "PROVIDER_RATE_LIMITED",
"insufficient_quota": "PROVIDER_RATE_LIMITED",
"not_found_error": "MODEL_NOT_FOUND",
"model_not_found": "MODEL_NOT_FOUND",
"invalid_request_error": "PROVIDER_INVALID_REQUEST",
"context_length_exceeded": "PROVIDER_INVALID_REQUEST",
}
mapped = mapping.get(code, "PROVIDER_UNAVAILABLE") if isinstance(code, str) else "PROVIDER_UNAVAILABLE"
return ProviderError(mapped, "Provider could not complete the request.")
def check_error(data: dict) -> None:
if data.get("error") is not None or data.get("type") == "error":
raise remote_error(data.get("error") or data)
class UsageTracker:
"""Merge cumulative snapshots, including partial usage updates."""
def __init__(self, input_key: str = "input_tokens", output_key: str = "output_tokens",
*, cache_tokens: bool = False) -> None:
self.input_key = input_key
self.output_key = output_key
self.cache_tokens = cache_tokens
self.counts: dict[str, int] = {}
def update(self, value: object) -> dict[str, int]:
usage = object_value(value)
keys = [self.input_key, self.output_key]
if self.cache_tokens:
keys += ["cache_creation_input_tokens", "cache_read_input_tokens"]
for key in keys:
if key in usage:
self.counts[key] = max(self.counts.get(key, 0), token_count(usage[key]))
inputs = self.counts.get(self.input_key, 0)
if self.cache_tokens:
inputs += sum(self.counts.get(key, 0) for key in keys[2:])
return {"input_tokens": inputs, "output_tokens": self.counts.get(self.output_key, 0)}
class EventStreamingMixin:
async def stream(self, request: ModelRequest) -> AsyncIterator[ModelEvent]:
sequence = 0
status = "completed"
try:
request, originals = prepare_tool_names(request)
# Closing the public iterator must synchronously close every nested iterator.
async with aclosing(self._events(request)) as events:
async for kind, data in events:
if kind == ModelEventType.tool_call_start and "name" in data:
data = {**data, "name": originals.get(data["name"], data["name"])}
if kind == ModelEventType.usage:
data = {**data, "total_tokens": data["input_tokens"] + data["output_tokens"]}
yield ModelEvent(event=kind, data=data, sequence=sequence,
timestamp=datetime.now(timezone.utc))
sequence += 1
except ProviderError as exc:
status = "failed"
yield ModelEvent(event=ModelEventType.error, sequence=sequence,
data={"code": exc.code, "message": exc.message},
timestamp=datetime.now(timezone.utc))
sequence += 1
except (ValueError, TypeError, KeyError, IndexError, AttributeError, OverflowError):
status = "failed"
error = invalid_response()
yield ModelEvent(event=ModelEventType.error, sequence=sequence,
data={"code": error.code, "message": error.message},
timestamp=datetime.now(timezone.utc))
sequence += 1
# CancelledError and GeneratorExit deliberately propagate without a Done event.
yield ModelEvent(event=ModelEventType.done, sequence=sequence,
data={"status": status},
timestamp=datetime.now(timezone.utc))
async def sse_objects(response: httpx.Response) -> AsyncIterator[dict]:
"""Read SSE frames, accepting the adjacent data lines used by some gateways."""
parts: list[str] = []
event_name = ""
def decode() -> dict:
value = "\n".join(parts)
if value.strip() == "[DONE]":
return {"type": "[DONE]"}
try:
data = object_value(json.loads(value))
except (ValueError, TypeError) as exc:
raise invalid_response() from exc
if event_name and "type" not in data:
data["type"] = event_name
check_error(data)
return data
async for line in response.aiter_lines():
if not line:
if parts:
yield decode()
parts = []
event_name = ""
elif line.startswith(":"):
continue
elif line.startswith("event:"):
if parts:
yield decode()
parts = []
event_name = line[6:].strip()
elif line.startswith("data:"):
if parts:
# Legacy compatible endpoints sometimes omit blank separators.
try:
json.loads("\n".join(parts))
except ValueError:
pass
else:
yield decode()
parts = []
event_name = ""
parts.append(line[5:].removeprefix(" "))
if parts:
yield decode()
class HTTPProviderMixin:
stream_path = "/chat/completions"
stream_format = "sse"
def _headers(self) -> dict[str, str]:
return {"Content-Type": "application/json"}
@staticmethod
def _status_error(exc: httpx.HTTPStatusError) -> ProviderError:
status = exc.response.status_code
code = {400: "PROVIDER_INVALID_REQUEST", 401: "PROVIDER_AUTH_FAILED",
403: "PROVIDER_AUTH_FAILED", 404: "MODEL_NOT_FOUND",
408: "PROVIDER_TIMEOUT", 413: "PROVIDER_INVALID_REQUEST",
422: "PROVIDER_INVALID_REQUEST", 429: "PROVIDER_RATE_LIMITED"}.get(
status, "PROVIDER_UNAVAILABLE")
return ProviderError(code, f"Provider returned HTTP {status}.")
async def _request(self, method: str, path: str, **kwargs) -> dict:
headers = self._headers()
try:
async with httpx.AsyncClient(timeout=self.timeout_seconds, transport=self.transport) as client:
response = await client.request(method, f"{self.base_url}{path}", headers=headers, **kwargs)
response.raise_for_status()
data = object_value(response.json())
check_error(data)
return data
except httpx.TimeoutException as exc:
raise ProviderError("PROVIDER_TIMEOUT", "Provider request timed out.") from exc
except httpx.HTTPStatusError as exc:
raise self._status_error(exc) from exc
except httpx.HTTPError as exc:
raise ProviderError("PROVIDER_UNAVAILABLE", "Provider is unavailable.") from exc
except (ValueError, TypeError) as exc:
raise invalid_response() from exc
async def _stream_json(self, payload: dict[str, object]) -> AsyncIterator[dict]:
headers = self._headers()
headers["Accept"] = "text/event-stream" if self.stream_format == "sse" else "application/x-ndjson"
try:
async with httpx.AsyncClient(timeout=self.timeout_seconds, transport=self.transport) as client:
async with client.stream("POST", f"{self.base_url}{self.stream_path}",
headers=headers, json=payload) as response:
response.raise_for_status()
if self.stream_format == "sse":
async with aclosing(sse_objects(response)) as objects:
async for data in objects:
yield data
else:
async for line in response.aiter_lines():
if line.strip():
data = object_value(json.loads(line))
check_error(data)
yield data
except httpx.TimeoutException as exc:
raise ProviderError("PROVIDER_TIMEOUT", "Provider request timed out.") from exc
except httpx.HTTPStatusError as exc:
raise self._status_error(exc) from exc
except httpx.HTTPError as exc:
raise ProviderError("PROVIDER_UNAVAILABLE", "Provider is unavailable.") from exc
except (ValueError, TypeError) as exc:
raise invalid_response() from exc
+88 -178
View File
@@ -1,16 +1,22 @@
from uuid import uuid4
import json
from collections.abc import AsyncIterator
from datetime import datetime, timezone
from contextlib import aclosing
from uuid import uuid4
import httpx
from app.contracts import ModelCapability, ModelEvent, ModelEventType, ModelInfo, ModelRequest
from app.contracts import MessageRole, ModelCapability, ModelEventType, ModelInfo, ModelRequest
from app.providers.base import ProviderError, ProviderToolCall, ProviderTurn
from app.providers.http_base import TurnStreamingMixin, decode_tool_arguments
from app.providers.tool_names import mapped_tool_names
from app.providers.http_base import (
EventStreamingMixin, HTTPProviderMixin, UsageTracker, decode_tool_arguments,
invalid_response, list_value, object_value, string_value, truncated_stream,
)
class OllamaProvider(TurnStreamingMixin):
class OllamaProvider(EventStreamingMixin, HTTPProviderMixin):
stream_path = "/api/chat"
stream_format = "jsonl"
def __init__(
self,
base_url: str = "http://127.0.0.1:11434",
@@ -21,126 +27,55 @@ class OllamaProvider(TurnStreamingMixin):
self.timeout_seconds = timeout_seconds
self.transport = transport
@mapped_tool_names
async def complete(self, request: ModelRequest) -> ProviderTurn:
messages = []
if request.system:
messages.append({"role": "system", "content": request.system})
for message in request.messages:
item: dict[str, object] = {
"role": message.role.value,
"content": message.content,
}
if message.tool_calls:
item["tool_calls"] = [
{
"function": {
"name": call.name,
"arguments": call.arguments,
}
}
for call in message.tool_calls
]
messages.append(item)
payload: dict[str, object] = {
"model": request.model,
"messages": messages,
"stream": False,
}
if request.tools:
payload["tools"] = [
{
"type": "function",
"function": {
"name": tool.name,
"description": tool.description,
"parameters": tool.parameters,
},
}
for tool in request.tools
]
data = await self._request("POST", "/api/chat", json=payload)
message = data.get("message") or {}
tool_calls = []
for raw_call in message.get("tool_calls") or []:
function = raw_call.get("function") or {}
tool_calls.append(
ProviderToolCall(
tool_call_id=raw_call.get("id") or f"call_{uuid4().hex}",
name=function.get("name") or "",
arguments=decode_tool_arguments(function.get("arguments", {})),
)
)
return ProviderTurn(
text=message.get("content") or None,
tool_calls=tool_calls,
input_tokens=int(data.get("prompt_eval_count") or 0),
output_tokens=int(data.get("eval_count") or 0),
data = await self._request("POST", self.stream_path, json=self._chat_payload(request, stream=False))
message = object_value(data.get("message"))
calls = [self._tool_call(raw) for raw in list_value(message.get("tool_calls", []))]
content = message.get("content")
if content is not None:
content = string_value(content)
return ProviderTurn(text=content or None, tool_calls=calls,
**UsageTracker("prompt_eval_count", "eval_count").update(data))
@staticmethod
def _tool_call(raw: object) -> ProviderToolCall:
call = object_value(raw)
function = object_value(call.get("function"))
return ProviderToolCall(
tool_call_id=string_value(call.get("id") or f"call_{uuid4().hex}"),
name=string_value(function.get("name"), nonempty=True),
arguments=decode_tool_arguments(function.get("arguments", {})),
)
async def list_models(self) -> list[ModelInfo]:
data = await self._request("GET", "/api/tags")
return [
ModelInfo(
model=item["name"],
display_name=item.get("name", ""),
capabilities=[ModelCapability.chat, ModelCapability.streaming],
)
for item in data.get("models", [])
if isinstance(item, dict) and item.get("name")
]
async def stream(self, request: ModelRequest) -> AsyncIterator[ModelEvent]:
payload = self._chat_payload(request, stream=True)
sequence = 0
def event(kind: ModelEventType, data: dict | None = None) -> ModelEvent:
nonlocal sequence
item = ModelEvent(
event=kind, sequence=sequence, data=data or {},
timestamp=datetime.now(timezone.utc),
)
sequence += 1
return item
try:
async for data in self._stream_json(payload):
message = data.get("message") or {}
async def _events(self, request: ModelRequest):
usage = UsageTracker("prompt_eval_count", "eval_count")
async with aclosing(self._stream_json(self._chat_payload(request, stream=True))) as chunks:
async for data in chunks:
message = object_value(data.get("message", {}))
if message.get("thinking"):
yield event(ModelEventType.thinking_delta, {"text": message["thinking"]})
yield ModelEventType.thinking_delta, {"text": string_value(message["thinking"])}
if message.get("content"):
yield event(ModelEventType.text_delta, {"text": message["content"]})
for raw_call in message.get("tool_calls") or []:
function = raw_call.get("function") or {}
call_id = raw_call.get("id") or f"call_{uuid4().hex}"
yield event(
ModelEventType.tool_call_start,
{"tool_call_id": call_id, "name": function.get("name") or ""},
)
yield event(
ModelEventType.tool_call_delta,
{
"tool_call_id": call_id,
"arguments_delta": json.dumps(
function.get("arguments") or {}, ensure_ascii=False
),
},
)
yield event(ModelEventType.tool_call_end, {"tool_call_id": call_id})
if data.get("done"):
yield event(
ModelEventType.usage,
{
"input_tokens": int(data.get("prompt_eval_count") or 0),
"output_tokens": int(data.get("eval_count") or 0),
},
)
yield event(ModelEventType.done)
except ProviderError as exc:
yield event(ModelEventType.error, {"code": exc.code, "message": exc.message})
yield event(ModelEventType.done)
yield ModelEventType.text_delta, {"text": string_value(message["content"])}
for raw in list_value(message.get("tool_calls", [])):
call = self._tool_call(raw)
yield ModelEventType.tool_call_start, {"tool_call_id": call.tool_call_id, "name": call.name}
yield ModelEventType.tool_call_delta, {
"tool_call_id": call.tool_call_id,
"arguments_delta": json.dumps(call.arguments, ensure_ascii=False),
}
yield ModelEventType.tool_call_end, {"tool_call_id": call.tool_call_id}
if "done" in data and not isinstance(data["done"], bool):
raise invalid_response()
if "prompt_eval_count" in data or "eval_count" in data or data.get("done"):
yield ModelEventType.usage, usage.update(data)
if data.get("done") is True:
return
raise truncated_stream()
def _chat_payload(self, request: ModelRequest, *, stream: bool) -> dict[str, object]:
messages = []
names: dict[str, str] = {}
if request.system:
messages.append({"role": "system", "content": request.system})
for message in request.messages:
@@ -150,53 +85,49 @@ class OllamaProvider(TurnStreamingMixin):
{"function": {"name": call.name, "arguments": call.arguments}}
for call in message.tool_calls
]
names.update({call.tool_call_id: call.name for call in message.tool_calls})
if message.role == MessageRole.tool:
name = message.name or names.get(message.tool_call_id or "")
if name:
item["tool_name"] = name
messages.append(item)
payload: dict[str, object] = {
"model": request.model, "messages": messages, "stream": stream
"model": request.model, "messages": messages, "stream": stream,
}
if request.tools:
payload["tools"] = [
{
"type": "function",
"function": {
"name": tool.name,
"description": tool.description,
"parameters": tool.parameters,
},
}
for tool in request.tools
{"type": "function", "function": {
"name": tool.name, "description": tool.description, "parameters": tool.parameters,
}} for tool in request.tools
]
options = {}
if request.temperature is not None:
options["temperature"] = request.temperature
if request.max_tokens is not None:
options["num_predict"] = request.max_tokens
if options:
payload["options"] = options
if request.response_format:
format_ = request.response_format
if format_.get("type") == "json_object":
payload["format"] = "json"
elif format_.get("type") == "json_schema":
payload["format"] = object_value(object_value(format_.get("json_schema")).get("schema"))
else:
payload["format"] = format_
return payload
async def _stream_json(self, payload: dict[str, object]) -> AsyncIterator[dict]:
try:
async with httpx.AsyncClient(
timeout=self.timeout_seconds, transport=self.transport
) as client:
async with client.stream(
"POST", f"{self.base_url}/api/chat", json=payload
) as response:
response.raise_for_status()
async for line in response.aiter_lines():
if not line.strip():
continue
try:
data = json.loads(line)
except json.JSONDecodeError as exc:
raise ProviderError(
"PROVIDER_INVALID_RESPONSE", "Ollama returned invalid JSONL."
) from exc
if isinstance(data, dict):
yield data
except httpx.TimeoutException as exc:
raise ProviderError("PROVIDER_TIMEOUT", "Ollama request timed out.") from exc
except httpx.HTTPStatusError as exc:
raise ProviderError(
"MODEL_NOT_FOUND" if exc.response.status_code == 404 else "PROVIDER_UNAVAILABLE",
f"Ollama returned HTTP {exc.response.status_code}.",
) from exc
except httpx.HTTPError as exc:
raise ProviderError("PROVIDER_UNAVAILABLE", "Ollama is unavailable.") from exc
async def list_models(self) -> list[ModelInfo]:
data = await self._request("GET", "/api/tags")
return [
ModelInfo(
model=string_value(item["name"]), display_name=item["name"],
capabilities=([ModelCapability.embedding] if "embed" in item["name"].lower()
else [ModelCapability.chat, ModelCapability.streaming]),
)
for item in list_value(data.get("models"))
if isinstance(item, dict) and isinstance(item.get("name"), str) and item["name"]
]
async def test_connection(self, model: str | None = None) -> tuple[bool, str]:
try:
@@ -206,24 +137,3 @@ class OllamaProvider(TurnStreamingMixin):
if model and model not in {item.model for item in models}:
return False, f"Model is not installed: {model}"
return True, f"Connected; discovered {len(models)} local model(s)."
async def _request(self, method: str, path: str, **kwargs) -> dict:
try:
async with httpx.AsyncClient(
timeout=self.timeout_seconds, transport=self.transport
) as client:
response = await client.request(method, f"{self.base_url}{path}", **kwargs)
response.raise_for_status()
data = response.json()
except httpx.TimeoutException as exc:
raise ProviderError("PROVIDER_TIMEOUT", "Ollama request timed out.") from exc
except httpx.HTTPStatusError as exc:
raise ProviderError(
"MODEL_NOT_FOUND" if exc.response.status_code == 404 else "PROVIDER_UNAVAILABLE",
f"Ollama returned HTTP {exc.response.status_code}.",
) from exc
except (httpx.HTTPError, ValueError) as exc:
raise ProviderError("PROVIDER_UNAVAILABLE", "Ollama is unavailable.") from exc
if not isinstance(data, dict):
raise ProviderError("PROVIDER_INVALID_RESPONSE", "Ollama returned non-object JSON.")
return data
+106 -209
View File
@@ -1,24 +1,20 @@
import json
from collections.abc import AsyncIterator
from datetime import datetime, timezone
from contextlib import aclosing
from uuid import uuid4
import httpx
from app.contracts import (
MessageRole,
ModelCapability,
ModelEvent,
ModelEventType,
ModelInfo,
ModelRequest,
)
from app.contracts import MessageRole, ModelCapability, ModelEventType, ModelInfo, ModelRequest
from app.providers.base import ProviderError, ProviderToolCall, ProviderTurn
from app.providers.credentials import CredentialResolver, CredentialStoreError
from app.providers.http_base import TurnStreamingMixin, decode_tool_arguments
from app.providers.tool_names import mapped_tool_names
from app.providers.http_base import (
EventStreamingMixin, HTTPProviderMixin, UsageTracker, decode_tool_arguments,
invalid_response, list_value, object_value, string_value, token_count, truncated_stream,
)
class OpenAICompatibleProvider(TurnStreamingMixin):
class OpenAICompatibleProvider(EventStreamingMixin, HTTPProviderMixin):
def __init__(
self,
base_url: str,
@@ -33,50 +29,37 @@ class OpenAICompatibleProvider(TurnStreamingMixin):
self.timeout_seconds = timeout_seconds
self.transport = transport
@mapped_tool_names
async def complete(self, request: ModelRequest) -> ProviderTurn:
payload = self._payload(request, stream=False)
data = await self._request("POST", "/chat/completions", json=payload)
try:
message = data["choices"][0]["message"]
except (KeyError, IndexError, TypeError) as exc:
raise ProviderError("PROVIDER_INVALID_RESPONSE", "Missing completion message.") from exc
tool_calls = []
for raw_call in message.get("tool_calls") or []:
function = raw_call.get("function") or {}
tool_calls.append(
ProviderToolCall(
tool_call_id=raw_call.get("id") or f"call_{uuid4().hex}",
name=function.get("name") or "",
arguments=decode_tool_arguments(function.get("arguments", "{}")),
)
)
usage = data.get("usage") or {}
return ProviderTurn(
text=message.get("content"),
tool_calls=tool_calls,
input_tokens=int(usage.get("prompt_tokens") or 0),
output_tokens=int(usage.get("completion_tokens") or 0),
)
data = await self._request("POST", self.stream_path, json=self._payload(request, stream=False))
choices = list_value(data.get("choices"))
if not choices:
raise invalid_response()
message = object_value(object_value(choices[0]).get("message"))
calls = []
for raw in list_value(message.get("tool_calls", [])):
raw = object_value(raw)
function = object_value(raw.get("function"))
calls.append(ProviderToolCall(
tool_call_id=string_value(raw.get("id") or f"call_{uuid4().hex}"),
name=string_value(function.get("name"), nonempty=True),
arguments=decode_tool_arguments(function.get("arguments", "{}")),
))
text = message.get("content")
if text is not None:
text = string_value(text)
usage = UsageTracker("prompt_tokens", "completion_tokens").update(data.get("usage") or {})
return ProviderTurn(text=text, tool_calls=calls, **usage)
def _payload(self, request: ModelRequest, *, stream: bool) -> dict[str, object]:
payload: dict[str, object] = {
"model": request.model,
"messages": self._messages(request),
"stream": stream,
"model": request.model, "messages": self._messages(request), "stream": stream,
}
if request.tools:
payload["tools"] = [
{
"type": "function",
"function": {
"name": tool.name,
"description": tool.description,
"parameters": tool.parameters,
},
}
for tool in request.tools
{"type": "function", "function": {
"name": tool.name, "description": tool.description, "parameters": tool.parameters,
}} for tool in request.tools
]
if request.temperature is not None:
payload["temperature"] = request.temperature
@@ -84,124 +67,81 @@ class OpenAICompatibleProvider(TurnStreamingMixin):
payload["max_tokens"] = request.max_tokens
if request.response_format is not None:
payload["response_format"] = request.response_format
if stream:
payload["stream_options"] = {"include_usage": True}
return payload
async def stream(self, request: ModelRequest) -> AsyncIterator[ModelEvent]:
sequence = 0
open_calls: dict[int, str] = {}
def event(kind: ModelEventType, data: dict | None = None) -> ModelEvent:
nonlocal sequence
item = ModelEvent(
event=kind,
sequence=sequence,
data=data or {},
timestamp=datetime.now(timezone.utc),
)
sequence += 1
return item
try:
async for data in self._stream_json(self._payload(request, stream=True)):
usage = data.get("usage") or {}
if usage:
yield event(
ModelEventType.usage,
{
"input_tokens": int(usage.get("prompt_tokens") or 0),
"output_tokens": int(usage.get("completion_tokens") or 0),
},
)
choices = data.get("choices") or []
async def _events(self, request: ModelRequest):
calls: dict[int, dict] = {}
usage = UsageTracker("prompt_tokens", "completion_tokens")
finished = False
seen = False
async with aclosing(self._stream_json(self._payload(request, stream=True))) as chunks:
async for data in chunks:
if data.get("type") == "[DONE]":
if not seen:
raise invalid_response()
finished = True
break
if data.get("usage") is not None:
yield ModelEventType.usage, usage.update(data["usage"])
choices = list_value(data.get("choices", []))
if not choices:
continue
choice = choices[0]
delta = choice.get("delta") or {}
seen = True
choice = object_value(choices[0])
delta = object_value(choice.get("delta") or {})
if delta.get("reasoning_content"):
yield event(
ModelEventType.thinking_delta,
{"text": delta["reasoning_content"]},
)
yield ModelEventType.thinking_delta, {"text": string_value(delta["reasoning_content"])}
if delta.get("content"):
yield event(ModelEventType.text_delta, {"text": delta["content"]})
for raw_call in delta.get("tool_calls") or []:
index = int(raw_call.get("index") or 0)
function = raw_call.get("function") or {}
call_id = raw_call.get("id") or open_calls.get(index) or f"call_{uuid4().hex}"
if index not in open_calls:
open_calls[index] = call_id
yield event(
ModelEventType.tool_call_start,
{"tool_call_id": call_id, "name": function.get("name") or ""},
)
if function.get("arguments"):
yield event(
ModelEventType.tool_call_delta,
{
"tool_call_id": open_calls[index],
"arguments_delta": function["arguments"],
},
)
if choice.get("finish_reason") == "tool_calls":
for call_id in open_calls.values():
yield event(
ModelEventType.tool_call_end, {"tool_call_id": call_id}
)
open_calls.clear()
for call_id in open_calls.values():
yield event(ModelEventType.tool_call_end, {"tool_call_id": call_id})
yield event(ModelEventType.done)
except ProviderError as exc:
yield event(ModelEventType.error, {"code": exc.code, "message": exc.message})
yield event(ModelEventType.done)
async def _stream_json(self, payload: dict[str, object]) -> AsyncIterator[dict]:
headers = self._headers()
try:
async with httpx.AsyncClient(
timeout=self.timeout_seconds, transport=self.transport
) as client:
async with client.stream(
"POST", f"{self.base_url}/chat/completions", headers=headers, json=payload
) as response:
response.raise_for_status()
async for line in response.aiter_lines():
if not line.startswith("data:"):
continue
value = line[5:].strip()
if not value or value == "[DONE]":
continue
try:
data = json.loads(value)
except json.JSONDecodeError as exc:
raise ProviderError(
"PROVIDER_INVALID_RESPONSE", "Provider returned invalid SSE JSON."
) from exc
if isinstance(data, dict):
yield data
except httpx.TimeoutException as exc:
raise ProviderError("PROVIDER_TIMEOUT", "Provider request timed out.") from exc
except httpx.HTTPStatusError as exc:
raise self._status_error(exc) from exc
except httpx.HTTPError as exc:
raise ProviderError("PROVIDER_UNAVAILABLE", "Provider is unavailable.") from exc
yield ModelEventType.text_delta, {"text": string_value(delta["content"])}
for raw in list_value(delta.get("tool_calls", [])):
raw = object_value(raw)
index = token_count(raw.get("index", 0))
function = object_value(raw.get("function") or {})
call = calls.setdefault(index, {"id": "", "name": "", "arguments": "", "started": False})
if raw.get("id"):
call["id"] = string_value(raw["id"])
if function.get("name"):
call["name"] += string_value(function["name"])
fragment = string_value(function.get("arguments", ""))
call["arguments"] += fragment
if not call["started"] and call["name"]:
call["id"] = call["id"] or f"call_{uuid4().hex}"
call["started"] = True
yield ModelEventType.tool_call_start, {"tool_call_id": call["id"], "name": call["name"]}
fragment = call["arguments"]
if call["started"] and fragment:
yield ModelEventType.tool_call_delta, {"tool_call_id": call["id"], "arguments_delta": fragment}
if choice.get("finish_reason"):
finished = True
if not finished:
raise truncated_stream()
for call in calls.values():
if not call["started"]:
raise invalid_response()
decode_tool_arguments(call["arguments"] or "{}")
yield ModelEventType.tool_call_end, {"tool_call_id": call["id"]}
async def list_models(self) -> list[ModelInfo]:
data = await self._request("GET", "/models")
return [
ModelInfo(
model=item["id"],
display_name=item["id"],
capabilities=[
ModelCapability.chat,
ModelCapability.tool_calling,
ModelCapability.streaming,
],
)
for item in data.get("data", [])
if isinstance(item, dict) and item.get("id")
]
return [ModelInfo(model=string_value(item["id"]), display_name=item["id"],
capabilities=self._model_capabilities(string_value(item["id"])))
for item in list_value(data.get("data"))
if isinstance(item, dict) and item.get("id")]
@staticmethod
def _model_capabilities(model: str) -> list[ModelCapability]:
# /models does not advertise capabilities. Avoid known non-chat families;
# these are discovery hints, not a guarantee of support by a gateway.
name = model.lower()
if "embed" in name or name.startswith(("bge-", "bge/")):
return [ModelCapability.embedding]
if any(marker in name for marker in (
"whisper", "tts", "transcri", "audio", "realtime", "dall-e", "image", "moderation", "rerank",
)):
return []
return [ModelCapability.chat]
async def test_connection(self, model: str | None = None) -> tuple[bool, str]:
try:
@@ -217,73 +157,30 @@ class OpenAICompatibleProvider(TurnStreamingMixin):
if request.system:
result.append({"role": "system", "content": request.system})
for message in request.messages:
item: dict[str, object] = {
"role": message.role.value,
"content": message.content,
}
item: dict[str, object] = {"role": message.role.value, "content": message.content}
if message.name:
item["name"] = message.name
if message.role == MessageRole.tool and message.tool_call_id:
item["tool_call_id"] = message.tool_call_id
if message.tool_calls:
item["tool_calls"] = [
{
"id": call.tool_call_id,
"type": "function",
"function": {
"name": call.name,
"arguments": json.dumps(call.arguments),
},
}
for call in message.tool_calls
{"id": call.tool_call_id, "type": "function", "function": {
"name": call.name, "arguments": json.dumps(call.arguments),
}} for call in message.tool_calls
]
result.append(item)
return result
async def _request(self, method: str, path: str, **kwargs) -> dict:
headers = self._headers()
try:
async with httpx.AsyncClient(
timeout=self.timeout_seconds, transport=self.transport
) as client:
response = await client.request(
method, f"{self.base_url}{path}", headers=headers, **kwargs
)
response.raise_for_status()
data = response.json()
except httpx.TimeoutException as exc:
raise ProviderError("PROVIDER_TIMEOUT", "Provider request timed out.") from exc
except httpx.HTTPStatusError as exc:
raise self._status_error(exc) from exc
except (httpx.HTTPError, ValueError) as exc:
raise ProviderError("PROVIDER_UNAVAILABLE", "Provider is unavailable.") from exc
if not isinstance(data, dict):
raise ProviderError("PROVIDER_INVALID_RESPONSE", "Provider returned non-object JSON.")
return data
def _headers(self) -> dict[str, str]:
headers = {"Content-Type": "application/json"}
try:
api_key = self.credentials.resolve(self.credential_id)
except CredentialStoreError as exc:
raise ProviderError(
"PROVIDER_CREDENTIAL_UNAVAILABLE",
"Credential could not be decrypted by the AI Core.",
) from exc
raise ProviderError("PROVIDER_CREDENTIAL_UNAVAILABLE",
"Credential could not be decrypted by the AI Core.") from exc
if self.credential_id and not api_key:
raise ProviderError(
"PROVIDER_CREDENTIAL_MISSING",
f'Credential "{self.credential_id}" is not available in the AI Core process.',
)
raise ProviderError("PROVIDER_CREDENTIAL_MISSING",
"Credential is not available in the AI Core process.")
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
return headers
@staticmethod
def _status_error(exc: httpx.HTTPStatusError) -> ProviderError:
code = {
401: "PROVIDER_AUTH_FAILED",
404: "MODEL_NOT_FOUND",
429: "PROVIDER_RATE_LIMITED",
}.get(exc.response.status_code, "PROVIDER_UNAVAILABLE")
return ProviderError(code, f"Provider returned HTTP {exc.response.status_code}.")
+168
View File
@@ -0,0 +1,168 @@
"""Native /responses adapter; stateless history uses function_call/output items."""
import json
from contextlib import aclosing
from app.contracts import MessageRole, ModelEventType, ModelRequest
from app.providers.base import ProviderError, ProviderToolCall, ProviderTurn
from app.providers.http_base import (
UsageTracker, check_error, decode_tool_arguments, invalid_response, list_value,
object_value, remote_error, string_value, token_count, truncated_stream,
)
from app.providers.openai_compatible import OpenAICompatibleProvider
from app.providers.tool_names import mapped_tool_names
class OpenAIResponsesProvider(OpenAICompatibleProvider):
stream_path = "/responses"
def _payload(self, request: ModelRequest, *, stream: bool) -> dict[str, object]:
inputs = []
for message in request.messages:
if message.role == MessageRole.tool:
if not message.tool_call_id:
raise ProviderError("PROVIDER_INVALID_REQUEST", "Tool result requires a call identifier.")
inputs.append({"type": "function_call_output", "call_id": message.tool_call_id,
"output": message.content})
continue
if message.content or not message.tool_calls:
inputs.append({"role": message.role.value, "content": message.content})
for call in message.tool_calls:
inputs.append({"type": "function_call", "call_id": call.tool_call_id,
"name": call.name, "arguments": json.dumps(call.arguments)})
payload: dict[str, object] = {"model": request.model, "input": inputs, "stream": stream}
if request.system:
payload["instructions"] = request.system
if request.tools:
payload["tools"] = [{"type": "function", "name": tool.name,
"description": tool.description, "parameters": tool.parameters}
for tool in request.tools]
if request.temperature is not None:
payload["temperature"] = request.temperature
if request.max_tokens is not None:
payload["max_output_tokens"] = request.max_tokens
if request.response_format is not None:
format_ = dict(request.response_format)
if format_.get("type") == "json_schema":
format_ = {"type": "json_schema", **object_value(format_.get("json_schema"))}
payload["text"] = {"format": format_}
return payload
@staticmethod
def _check_response(data: dict) -> None:
check_error(data)
status = data.get("status")
if status == "incomplete":
raise ProviderError("PROVIDER_INCOMPLETE_RESPONSE", "Provider response is incomplete.")
if status == "failed":
raise remote_error(data.get("error"))
if status is not None and status != "completed":
raise invalid_response()
@mapped_tool_names
async def complete(self, request: ModelRequest) -> ProviderTurn:
data = await self._request("POST", self.stream_path, json=self._payload(request, stream=False))
self._check_response(data)
texts = []
calls = []
for raw in list_value(data.get("output")):
item = object_value(raw)
if item.get("type") == "message":
for raw_part in list_value(item.get("content")):
part = object_value(raw_part)
if part.get("type") == "output_text":
texts.append(string_value(part.get("text")))
elif part.get("type") == "refusal":
texts.append(string_value(part.get("refusal")))
elif item.get("type") == "function_call":
calls.append(ProviderToolCall(
tool_call_id=string_value(item.get("call_id"), nonempty=True),
name=string_value(item.get("name"), nonempty=True),
arguments=decode_tool_arguments(item.get("arguments")),
))
return ProviderTurn(text="".join(texts) or None, tool_calls=calls,
**UsageTracker().update(data.get("usage") or {}))
async def _events(self, request: ModelRequest):
calls: dict[int, dict] = {}
usage = UsageTracker()
def finish_call(index: int, final: object = None):
call = calls[index]
if call["ended"]:
return []
events = []
if final is not None:
arguments = string_value(final)
if not arguments.startswith(call["arguments"]):
raise invalid_response()
remainder = arguments[len(call["arguments"]):]
if remainder:
events.append((ModelEventType.tool_call_delta,
{"tool_call_id": call["id"], "arguments_delta": remainder}))
call["arguments"] = arguments
decode_tool_arguments(call["arguments"])
call["ended"] = True
events.append((ModelEventType.tool_call_end, {"tool_call_id": call["id"]}))
return events
async with aclosing(self._stream_json(self._payload(request, stream=True))) as chunks:
async for data in chunks:
kind = string_value(data.get("type"), nonempty=True)
if kind in {"response.failed", "response.incomplete"}:
response = object_value(data.get("response"))
self._check_response({**response, "status": kind.split(".")[1]})
elif kind in {"response.output_text.delta", "response.refusal.delta"}:
yield ModelEventType.text_delta, {"text": string_value(data.get("delta"))}
elif kind in {"response.reasoning_summary_text.delta", "response.reasoning_text.delta"}:
yield ModelEventType.thinking_delta, {"text": string_value(data.get("delta"))}
elif kind in {"response.output_item.added", "response.output_item.done"}:
item = object_value(data.get("item"))
if item.get("type") != "function_call":
continue
index = token_count(data.get("output_index"))
call_id = string_value(item.get("call_id"), nonempty=True)
name = string_value(item.get("name"), nonempty=True)
if index not in calls:
calls[index] = {"id": call_id, "name": name, "arguments": "", "ended": False,
"item_id": item.get("id")}
yield ModelEventType.tool_call_start, {"tool_call_id": call_id, "name": name}
elif calls[index]["id"] != call_id or calls[index]["name"] != name:
raise invalid_response()
if kind == "response.output_item.done":
for event in finish_call(index, item.get("arguments")):
yield event
elif item.get("arguments"):
arguments = string_value(item["arguments"])
calls[index]["arguments"] += arguments
yield ModelEventType.tool_call_delta, {"tool_call_id": call_id, "arguments_delta": arguments}
elif kind in {"response.function_call_arguments.delta", "response.function_call_arguments.done"}:
index = token_count(data.get("output_index"))
call = calls.get(index)
if call is None or (data.get("item_id") and call["item_id"] != data["item_id"]):
raise invalid_response()
if kind.endswith(".done"):
for event in finish_call(index, data.get("arguments")):
yield event
else:
if call["ended"]:
raise invalid_response()
fragment = string_value(data.get("delta"))
call["arguments"] += fragment
yield ModelEventType.tool_call_delta, {"tool_call_id": call["id"], "arguments_delta": fragment}
elif kind == "response.completed":
response = object_value(data.get("response"))
self._check_response(response)
if any(not call["ended"] for call in calls.values()):
raise truncated_stream()
if response.get("usage") is not None:
yield ModelEventType.usage, usage.update(response["usage"])
return
elif kind == "[DONE]":
raise truncated_stream()
elif kind in {"response.created", "response.in_progress"}:
response = object_value(data.get("response"))
check_error(response)
if response.get("usage") is not None:
yield ModelEventType.usage, usage.update(response["usage"])
raise truncated_stream()
+52 -1
View File
@@ -1,5 +1,10 @@
from dataclasses import dataclass
from time import perf_counter
from pathlib import Path
from app.config import get_settings
from app.database.db import connect
from app.errors import ApiError
from app.contracts import ModelInfo, ProviderConfig, ProviderTestResponse
from app.providers.base import ModelProvider
@@ -16,20 +21,64 @@ class RegisteredProvider:
class ProviderRegistry:
def __init__(self) -> None:
def __init__(self, factory=None) -> None:
self._providers: dict[str, RegisteredProvider] = {}
self._factory = factory
self._loaded_path: Path | None = None
def _restore(self) -> None:
if self._factory is None or self._loaded_path == get_settings().db_path:
return
conn = connect()
try:
conn.execute("CREATE TABLE IF NOT EXISTS provider_configs (provider_id TEXT PRIMARY KEY, config_json TEXT NOT NULL)")
restored = {}
for row in conn.execute("SELECT config_json FROM provider_configs"):
config = ProviderConfig.model_validate_json(row["config_json"])
if config.provider_id == "mock":
raise ValueError("reserved provider")
restored[config.provider_id] = RegisteredProvider(config, self._factory.build(config))
if "mock" in self._providers:
restored["mock"] = self._providers["mock"]
self._providers = restored
self._loaded_path = get_settings().db_path
except (ValueError, TypeError) as exc:
raise ApiError(500, "PROVIDER_STORAGE_INVALID", "Saved provider configuration could not be loaded.") from exc
finally:
conn.close()
def _save(self, config: ProviderConfig) -> None:
if self._factory is None or config.provider_id == "mock":
return
conn = connect()
try:
conn.execute("INSERT OR REPLACE INTO provider_configs VALUES (?, ?)", (config.provider_id, config.model_dump_json()))
finally:
conn.close()
def register(self, config: ProviderConfig, adapter: ModelProvider) -> None:
if config.provider_id != "mock":
self._restore()
if config.provider_id in self._providers:
raise ValueError(f"Provider already registered: {config.provider_id}")
self._save(config)
self._providers[config.provider_id] = RegisteredProvider(config=config, adapter=adapter)
def unregister(self, provider_id: str) -> None:
self._restore()
if self._factory is not None:
conn = connect()
try:
conn.execute("DELETE FROM provider_configs WHERE provider_id = ?", (provider_id,))
finally:
conn.close()
self._providers.pop(provider_id, None)
def replace(self, config: ProviderConfig, adapter: ModelProvider) -> None:
self._restore()
if config.provider_id not in self._providers:
raise ProviderNotFoundError(config.provider_id)
self._save(config)
self._providers[config.provider_id] = RegisteredProvider(config=config, adapter=adapter)
def get(self, provider_id: str) -> RegisteredProvider:
@@ -39,12 +88,14 @@ class ProviderRegistry:
return provider
def get_any(self, provider_id: str) -> RegisteredProvider:
self._restore()
try:
return self._providers[provider_id]
except KeyError as exc:
raise ProviderNotFoundError(provider_id) from exc
def list_configs(self) -> list[ProviderConfig]:
self._restore()
return [item.config.model_copy(deep=True) for item in self._providers.values()]
async def list_models(self, provider_id: str) -> list[ModelInfo]:
+287
View File
@@ -0,0 +1,287 @@
"""Capability routing: validated remote results, then an explicit local backend.
Phase E supplies HTTP adapters and injectable local contracts. Hash embeddings are
still a development placeholder; speech models are installed in phase F.
"""
from __future__ import annotations
import hashlib
import json
import math
from dataclasses import dataclass
from pathlib import Path
from typing import Protocol
import httpx
from app.contracts import (
EmbeddingResult, LocalBackendStatus, ModelBinding, ModelRoutingConfig,
ModelRoutingResponse, ProviderType, SpeakerMatchResult,
)
from app.database.db import connect, transaction
from app.errors import ApiError
from app.providers.base import ProviderError
from app.providers.credentials import CredentialResolver, CredentialStoreError
from app.providers.registry import ProviderNotFoundError, ProviderRegistry
from app.retrieval.embedding import EmbeddingProvider, HashEmbeddingProvider
CAPABILITIES = ("embedding", "transcription", "speaker_matching")
HTTP_TYPES = {ProviderType.openai_chat, ProviderType.openai_compatible}
MAX_MEDIA_BYTES = 25 * 1024 * 1024
MAX_RESPONSE_BYTES = 16 * 1024 * 1024
class LocalSpeechBackend(Protocol):
available: bool
async def transcribe(self, source: Path, language: str | None) -> str: ...
async def match(self, source: Path, reference: Path) -> float: ...
class PendingSpeechBackend:
available = False
async def transcribe(self, source: Path, language: str | None) -> str:
raise ProviderError("LOCAL_MODEL_NOT_INSTALLED", "本地音频转写模型尚未安装,将在阶段 F 接入。")
async def match(self, source: Path, reference: Path) -> float:
raise ProviderError("LOCAL_MODEL_NOT_INSTALLED", "本地声纹模型尚未安装,将在阶段 F 接入。")
@dataclass(frozen=True)
class RoutedTranscript:
text: str
source: str
fallback_reason: str | None = None
def invalid_response() -> ProviderError:
return ProviderError("PROVIDER_INVALID_RESPONSE", "Model API returned an invalid result.")
def finite_number(value: object) -> bool:
if type(value) not in (int, float):
return False
try:
return math.isfinite(value)
except (OverflowError, ValueError):
return False
class ModelRoutingService:
def __init__(self, providers: ProviderRegistry, credentials: CredentialResolver, *,
local_embedding: EmbeddingProvider | None = None,
local_speech: LocalSpeechBackend | None = None,
transport: httpx.AsyncBaseTransport | None = None) -> None:
self.providers = providers
self.credentials = credentials
self.local_embedding = local_embedding or HashEmbeddingProvider()
self.local_speech = local_speech or PendingSpeechBackend()
self.transport = transport
@staticmethod
def _connection():
conn = connect()
conn.execute("CREATE TABLE IF NOT EXISTS model_routing (id INTEGER PRIMARY KEY CHECK(id=1), config_json TEXT NOT NULL)")
return conn
def configuration(self) -> ModelRoutingConfig:
conn = self._connection()
try:
row = conn.execute("SELECT config_json FROM model_routing WHERE id=1").fetchone()
return ModelRoutingConfig.model_validate_json(row[0]) if row else ModelRoutingConfig()
except ValueError as exc:
raise ApiError(500, "MODEL_ROUTING_STORAGE_INVALID", "Saved model routing could not be loaded.") from exc
finally:
conn.close()
def describe(self) -> ModelRoutingResponse:
return ModelRoutingResponse(config=self.configuration(), local_backends=[
LocalBackendStatus(capability="embedding", status="placeholder" if isinstance(self.local_embedding, HashEmbeddingProvider) else "ready",
message="当前为 hash-v1 确定性占位向量,真实本地语义模型尚未集成。" if isinstance(self.local_embedding, HashEmbeddingProvider) else "本地 Embedding 模型已就绪。"),
*[LocalBackendStatus(capability=capability, status="ready" if self.local_speech.available else "not_installed",
message="本地模型已就绪。" if self.local_speech.available else "阶段 F 接入本地模型;当前保留回退接口。")
for capability in ("transcription", "speaker_matching")],
])
def update(self, config: ModelRoutingConfig) -> ModelRoutingResponse:
for capability in CAPABILITIES:
binding = getattr(config, capability)
if binding:
try:
provider = self.providers.get_any(binding.provider_id).config
except ProviderNotFoundError as exc:
raise ApiError(422, "PROVIDER_NOT_FOUND", "请选择已保存的提供商。") from exc
if provider.provider_type not in HTTP_TYPES:
raise ApiError(422, "MODEL_ROUTING_PROTOCOL_UNSUPPORTED", "该能力当前需要 OpenAI Compatible HTTP 接口。")
conn = self._connection()
try:
with transaction(conn):
row = conn.execute("SELECT config_json FROM model_routing WHERE id=1").fetchone()
current = ModelRoutingConfig.model_validate_json(row[0]) if row else ModelRoutingConfig()
if current.version != config.version:
raise ApiError(409, "MODEL_ROUTING_VERSION_CONFLICT", "配置已更新,请重新加载后再保存。")
saved = config.model_copy(update={"version": config.version + 1})
conn.execute("INSERT OR REPLACE INTO model_routing VALUES (1, ?)", (saved.model_dump_json(),))
finally:
conn.close()
return self.describe()
def uses_provider(self, provider_id: str) -> bool:
config = self.configuration()
return any(binding and binding.provider_id == provider_id for binding in
(getattr(config, name) for name in CAPABILITIES))
def _remote(self, binding: ModelBinding) -> tuple[str, dict[str, str]]:
try:
provider = self.providers.get(binding.provider_id).config
except ProviderNotFoundError as exc:
raise ProviderError("PROVIDER_UNAVAILABLE", "Configured provider is unavailable.") from exc
if provider.provider_type not in HTTP_TYPES:
raise ProviderError("PROVIDER_CAPABILITY_UNSUPPORTED", "Provider does not support this HTTP capability.")
try:
key = self.credentials.resolve(provider.credential_id)
except CredentialStoreError as exc:
raise ProviderError("PROVIDER_CREDENTIAL_UNAVAILABLE", "Provider credential is unavailable.") from exc
if provider.credential_id and not key:
raise ProviderError("PROVIDER_CREDENTIAL_MISSING", "Provider credential is not configured.")
url = (provider.base_url or "https://api.openai.com/v1").rstrip("/") + binding.endpoint
return url, {"Authorization": f"Bearer {key}"} if key else {}
async def _request(self, binding: ModelBinding, *, remote: tuple[str, dict[str, str]] | None = None, **kwargs) -> tuple[dict, str]:
url, headers = remote or self._remote(binding)
try:
async with httpx.AsyncClient(timeout=30, transport=self.transport) as client:
async with client.stream("POST", url, headers=headers, **kwargs) as response:
response.raise_for_status()
body = bytearray()
async for chunk in response.aiter_bytes():
body.extend(chunk)
if len(body) > MAX_RESPONSE_BYTES:
raise invalid_response()
data = json.loads(body)
except httpx.TimeoutException as exc:
raise ProviderError("PROVIDER_TIMEOUT", "Model API timed out.") from exc
except httpx.HTTPStatusError as exc:
code = {401: "PROVIDER_AUTH_FAILED", 403: "PROVIDER_AUTH_FAILED", 404: "MODEL_NOT_FOUND", 429: "PROVIDER_RATE_LIMITED"}.get(exc.response.status_code, "PROVIDER_UNAVAILABLE")
raise ProviderError(code, f"Model API returned HTTP {exc.response.status_code}.") from exc
except (httpx.HTTPError, httpx.InvalidURL) as exc:
raise ProviderError("PROVIDER_UNAVAILABLE", "Model API is unavailable.") from exc
except (ValueError, UnicodeError) as exc:
raise invalid_response() from exc
if not isinstance(data, dict) or data.get("error"):
raise invalid_response()
return data, url
async def embed(self, texts: list[str]) -> EmbeddingResult:
binding = self.configuration().embedding
reason = None
if binding and texts:
try:
vectors = []
dimension = binding.dimensions
# Freeze the origin across batches, even if the user edits the provider.
remote = self._remote(binding)
for start in range(0, len(texts), 32):
batch = texts[start:start + 32]
payload = {"model": binding.model, "input": batch, "encoding_format": "float"}
if binding.dimensions is not None:
payload["dimensions"] = binding.dimensions
data, url = await self._request(binding, remote=remote, json=payload)
items = data.get("data")
if not isinstance(items, list) or len(items) != len(batch):
raise invalid_response()
indexed = {}
for item in items:
if not isinstance(item, dict):
raise invalid_response()
index, vector = item.get("index"), item.get("embedding")
if type(index) is not int or index in indexed or not 0 <= index < len(batch):
raise invalid_response()
if not isinstance(vector, list) or not 1 <= len(vector) <= 16384:
raise invalid_response()
if any(not finite_number(value) for value in vector):
raise invalid_response()
dimension = dimension or len(vector)
norm = math.hypot(*vector)
if len(vector) != dimension or not norm or not math.isfinite(norm):
raise invalid_response()
indexed[index] = [value / norm for value in vector]
vectors.extend(indexed[index] for index in range(len(batch)))
identity = json.dumps([url, binding.model, dimension], separators=(",", ":"))
return EmbeddingResult(vectors=vectors, source="api", dimensions=dimension,
model_id="api-" + hashlib.sha256(identity.encode()).hexdigest())
except ProviderError as exc:
reason = exc.code
vectors = await self.local_embedding.embed_documents(texts)
return EmbeddingResult(vectors=vectors, source="local", model_id=self.local_embedding.model_id,
dimensions=self.local_embedding.dim, fallback_reason=reason)
@staticmethod
def _media_file(path: Path):
try:
handle = path.open("rb")
except OSError as exc:
raise ApiError(404, "ATTACHMENT_NOT_FOUND", "Audio attachment was not found.") from exc
import os
if not 0 < os.fstat(handle.fileno()).st_size <= MAX_MEDIA_BYTES:
handle.close()
raise ApiError(413, "ATTACHMENT_TOO_LARGE", "Audio attachment must be between 1 byte and 25 MiB.")
return handle
async def transcribe(self, source: Path, language: str | None) -> RoutedTranscript:
binding = self.configuration().transcription
if binding is None:
with self._media_file(source):
pass
reason = None
if binding:
try:
fields = {"model": binding.model}
if language:
fields["language"] = language
with self._media_file(source) as handle:
data, _ = await self._request(binding, data=fields,
files={"file": (source.name, handle, "application/octet-stream")})
text = data.get("text")
if not isinstance(text, str) or not text.strip():
raise invalid_response()
return RoutedTranscript(text=text, source="api")
except ProviderError as exc:
reason = exc.code
try:
text = await self.local_speech.transcribe(source, language)
if not isinstance(text, str) or not text.strip():
raise ProviderError("LOCAL_MODEL_INVALID_RESPONSE", "Local transcription was empty.")
return RoutedTranscript(text=text, source="local", fallback_reason=reason)
except ProviderError as exc:
raise ApiError(503, exc.code, exc.message, {"fallback_reason": reason}) from exc
async def match_speakers(self, source: Path, reference: Path) -> SpeakerMatchResult:
binding = self.configuration().speaker_matching
if binding is None:
with self._media_file(source), self._media_file(reference):
pass
reason = None
if binding:
try:
# Explicit application contract, not an OpenAI-standard endpoint.
with self._media_file(source) as audio, self._media_file(reference) as sample:
data, _ = await self._request(binding, data={"model": binding.model}, files={
"file": (source.name, audio, "application/octet-stream"),
"reference_file": (reference.name, sample, "application/octet-stream"),
})
score = data.get("score")
if not finite_number(score) or not 0 <= score <= 1:
raise invalid_response()
return SpeakerMatchResult(score=score, source="api")
except ProviderError as exc:
reason = exc.code
try:
score = await self.local_speech.match(source, reference)
if not finite_number(score) or not 0 <= score <= 1:
raise ProviderError("LOCAL_MODEL_INVALID_RESPONSE", "Local speaker matching was invalid.")
return SpeakerMatchResult(score=score, source="local", fallback_reason=reason)
except ProviderError as exc:
raise ApiError(503, exc.code, exc.message, {"fallback_reason": reason}) from exc
+47
View File
@@ -0,0 +1,47 @@
"""Keep internal namespaced tools compatible with providers' 64-character names."""
import hashlib
import re
from functools import wraps
from app.contracts import MessageRole, ModelRequest
def prepare_tool_names(request: ModelRequest) -> tuple[ModelRequest, dict[str, str]]:
names = {tool.name for tool in request.tools}
for message in request.messages:
names.update(call.name for call in message.tool_calls)
if message.role == MessageRole.tool and message.name:
names.add(message.name)
mapping = {name: name for name in names if re.fullmatch(r"[A-Za-z0-9_-]{1,64}", name)}
used = set(mapping)
for name in sorted(names - mapping.keys()):
salt = 0
while True:
alias = "tool_" + hashlib.sha256(f"{name}:{salt}".encode()).hexdigest()[:56]
if alias not in used:
break
salt += 1
mapping[name] = alias
used.add(alias)
if all(name == alias for name, alias in mapping.items()):
return request, {}
wire = request.model_copy(deep=True)
for tool in wire.tools:
tool.name = mapping[tool.name]
for message in wire.messages:
for call in message.tool_calls:
call.name = mapping[call.name]
if message.role == MessageRole.tool and message.name:
message.name = mapping[message.name]
return wire, {alias: name for name, alias in mapping.items()}
def mapped_tool_names(complete):
@wraps(complete)
async def wrapped(self, request: ModelRequest):
wire, originals = prepare_tool_names(request)
turn = await complete(self, wire)
for call in turn.tool_calls:
call.name = originals.get(call.name, call.name)
return turn
return wrapped