From e86809b23867cf19a7bee259e60ba9b051c66f68 Mon Sep 17 00:00:00 2001 From: KiriAky 107 Date: Wed, 16 Sep 2026 22:05:09 +0800 Subject: [PATCH] =?UTF-8?q?feat(audio):=20=E4=BF=AE=E5=A4=8D=E8=AF=B4?= =?UTF-8?q?=E8=AF=9D=E4=BA=BA=E8=81=9A=E7=B1=BB=E5=B9=B6=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E8=87=B30.4.0-alpha.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +- backend/app/local_models/worker.py | 100 ++++++++++++++++-- backend/app/services/transcription_service.py | 2 + backend/tests/test_speaker_clustering.py | 38 +++++++ .../多模态管线与模型运行开发说明.md | 2 +- frontend/package.json | 2 +- frontend/src-tauri/Cargo.lock | 2 +- frontend/src-tauri/Cargo.toml | 2 +- frontend/src-tauri/tauri.conf.json | 2 +- frontend/src/features/media/MediaView.vue | 1 + frontend/src/features/vault/VaultEntry.vue | 2 +- frontend/src/services/themeImport.spec.ts | 2 +- server sync/Dockerfile | 2 +- server sync/README.md | 6 +- server sync/compose.yaml | 8 +- server sync/console/package.json | 2 +- server sync/pyproject.toml | 2 +- server sync/uv.lock | 2 +- 18 files changed, 151 insertions(+), 30 deletions(-) create mode 100644 backend/tests/test_speaker_clustering.py diff --git a/README.md b/README.md index e47be5b..c3d5785 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ OpenNexus 是一款本地优先的 AI 笔记与知识中枢。它将 Markdown Vault、全文与向量检索、知识库问答、可审计 Agent、扩展系统和多设备同步整合在一个桌面应用中。笔记与索引由用户掌控；需要模型或同步服务时，再按需连接本地或远程服务。 -当前发布版本为 **0.3.2-alpha.1**，主要支持 Windows x64。Alpha 版本仍处于快速迭代阶段，升级前请备份 Vault。 +当前发布版本为 **0.4.0-alpha.1**，主要支持 Windows x64。Alpha 版本仍处于快速迭代阶段，升级前请备份 Vault。 ## 主要能力 @@ -34,7 +34,7 @@ flowchart LR ## 使用发布包 -本版提供 Windows x64 EXE 安装包和独立的 Server Sync 包，下载入口见 [v0.3.2-alpha.1 发布页](https://gitea.kronecker.cc/Kronecker/NotesAgentic/releases/tag/v0.3.2-alpha.1)。发布页同时附带 `SHA256.json`，用于核对文件完整性。 +本版提供 Windows x64 EXE 安装包和独立的 Server Sync 包，下载入口见 [v0.4.0-alpha.1 发布页](https://gitea.kronecker.cc/Kronecker/NotesAgentic/releases/tag/v0.4.0-alpha.1)。发布页同时附带 `SHA256.json`，用于核对文件完整性。 安装包不包含任何 Vault 或用户数据，也不预装已下载的社区主题、本地模型权重、CUDA 与 PyTorch 运行时。相关功能仍完整保留；需要时可在客户端内按需安装主题、选择模型或配置 CUDA 环境。程序自带的基础界面样式属于客户端资源，不视为社区主题。同一 Windows 用户下升级安装会继续使用 `%APPDATA%\cc.kronecker.notesagent` 中的既有配置和索引，以及用户此前选择的外部 Vault。 diff --git a/backend/app/local_models/worker.py b/backend/app/local_models/worker.py index 0102dc6..663cf28 100644 --- a/backend/app/local_models/worker.py +++ b/backend/app/local_models/worker.py @@ -115,6 +115,91 @@ def voice_embedding(model, audio, device): return torch.nn.functional.normalize(vector, dim=0) +def _normalized_vector(values): + """把声纹向量转成普通列表并归一化，便于在无 PyTorch 的 API 测试环境中验证聚类。""" + import math + values = [float(value) for value in values] + norm = math.sqrt(sum(value * value for value in values)) + if not values or not math.isfinite(norm) or norm <= 1e-12: + raise ValueError("Invalid speaker embedding") + return [value / norm for value in values] + + +def _similarity(left, right): + return sum(a * b for a, b in zip(left, right, strict=True)) + + +def cluster_speaker_embeddings(embeddings, segments, *, threshold=0.36): + """聚类片段声纹，并把过短片段交给相邻的稳定说话人。 + + 质心在每次接收新样本后更新，避免第一段永久决定整簇。持续时间不超过 + 3 秒的孤立单例通常是停顿处的语气词；将它并入最相近的已有稳定簇， + 同时保留由多个片段支持的第三位及更多说话人。 + """ + if len(embeddings) != len(segments): + raise ValueError("Speaker embeddings and segments must have the same length") + vectors = [None if value is None else _normalized_vector(value) for value in embeddings] + assignments = [None] * len(vectors) + clusters = [] + for index, vector in enumerate(vectors): + if vector is None: + continue + similarities = [_similarity(vector, cluster["centroid"]) for cluster in clusters] + best = max(range(len(similarities)), key=similarities.__getitem__) if similarities else None + if best is None or similarities[best] < threshold: + best = len(clusters) + clusters.append({"members": [], "sum": [0.0] * len(vector), "centroid": vector}) + cluster = clusters[best] + cluster["members"].append(index) + cluster["sum"] = [total + value for total, value in zip(cluster["sum"], vector, strict=True)] + cluster["centroid"] = _normalized_vector(cluster["sum"]) + assignments[index] = best + + # 短语气词可能形成只有一个片段的离群簇。仅合并短单例，不吞掉由多个 + # 片段支持的真实少数说话人。 + stable = [index for index, cluster in enumerate(clusters) if len(cluster["members"]) > 1] + for index, cluster in enumerate(clusters): + member = cluster["members"][0] if len(cluster["members"]) == 1 else None + if member is None or not stable: + continue + duration = float(segments[member]["end_time"]) - float(segments[member]["start_time"]) + if duration > 3.0: + continue + target = max(stable, key=lambda other: _similarity(cluster["centroid"], clusters[other]["centroid"])) + assignments[member] = target + + # 没有足够语音生成声纹的短片段继承时间上最近的稳定标签。同一说话人 + # 两个片段之间的语气词会优先落回该说话人。 + labeled = [index for index, value in enumerate(assignments) if value is not None] + for index, value in enumerate(assignments): + if value is not None or not labeled: + continue + previous = next((item for item in reversed(labeled) if item < index), None) + following = next((item for item in labeled if item > index), None) + if previous is not None and following is not None and assignments[previous] == assignments[following]: + assignments[index] = assignments[previous] + continue + candidates = [] + if previous is not None: + distance = max(0.0, float(segments[index]["start_time"]) - float(segments[previous]["end_time"])) + candidates.append((distance, 0, assignments[previous])) + if following is not None: + distance = max(0.0, float(segments[following]["start_time"]) - float(segments[index]["end_time"])) + candidates.append((distance, 1, assignments[following])) + assignments[index] = min(candidates)[2] if candidates else None + + # 合并后按首次出现顺序重新编号，避免 speaker_1、speaker_3 这样的空洞 ID。 + remap = {} + speakers = [] + for value in assignments: + if value is None: + speakers.append(None) + continue + remap.setdefault(value, len(remap) + 1) + speakers.append(f"speaker_{remap[value]}") + return speakers + + class CudaInitializationError(RuntimeError): pass @@ -189,20 +274,15 @@ def run(request): model = speaker_model(path, device) loaded = time.monotonic() audio = decode(payload["source"]) - centroids, speakers = [], [] + embeddings = [] for segment in payload["segments"]: sample = audio[int(segment["start_time"] * 16000):int(segment["end_time"] * 16000)] if len(sample) < 16000: - speakers.append(None) + embeddings.append(None) continue - vector = voice_embedding(model, sample, device) - similarities = [float(torch.dot(vector, c)) for c in centroids] - best = max(range(len(similarities)), key=similarities.__getitem__) if similarities else None - if best is None or similarities[best] < 0.36: - best = len(centroids) - centroids.append(vector) - speakers.append(f"speaker_{best + 1}") - result = {"speakers": speakers} + embeddings.append(voice_embedding(model, sample, device).tolist()) + speakers = cluster_speaker_embeddings(embeddings, payload["segments"]) + result = {"speakers": speakers, "unassigned_segments": sum(speaker is None for speaker in speakers)} else: raise ValueError("Unknown inference operation") return {"result": result, "usage": usage, "audio_seconds": audio_seconds, "diagnostics": {"requested_device": requested, "actual_device": device, diff --git a/backend/app/services/transcription_service.py b/backend/app/services/transcription_service.py index 0202853..8b5e68b 100644 --- a/backend/app/services/transcription_service.py +++ b/backend/app/services/transcription_service.py @@ -174,6 +174,8 @@ async def _execute(job_id, request, routing=None): for segment, speaker in zip(job.segments, result["speakers"], strict=True): segment.speaker = speaker job.warnings.append("DIARIZATION_SEGMENT_LEVEL") + if result.get("unassigned_segments"): + job.warnings.append("DIARIZATION_PARTIAL") except ProviderError: job.warnings.append("DIARIZATION_UNAVAILABLE") else: diff --git a/backend/tests/test_speaker_clustering.py b/backend/tests/test_speaker_clustering.py new file mode 100644 index 0000000..e201e2d --- /dev/null +++ b/backend/tests/test_speaker_clustering.py @@ -0,0 +1,38 @@ +from app.local_models.worker import cluster_speaker_embeddings + + +def segment(start, end): + return {"start_time": start, "end_time": end} + + +def test_centroid_updates_allow_one_speaker_to_drift(): + speakers = cluster_speaker_embeddings( + [[1, 0], [0.8, 0.6], [0.55, 0.835]], + [segment(0, 2), segment(2, 4), segment(4, 6)], + threshold=0.7, + ) + assert speakers == ["speaker_1"] * 3 + + +def test_short_segments_and_short_singleton_join_stable_neighbors(): + speakers = cluster_speaker_embeddings( + [[1, 0], None, [0.98, 0.1], [0, 1], [-0.9, -0.1], [0.1, 0.99]], + [segment(0, 2), segment(2, 2.4), segment(2.4, 5), segment(5, 8), segment(8, 9.5), segment(9.5, 12)], + ) + assert speakers[0] == speakers[1] == speakers[2] == "speaker_1" + assert speakers[3] == speakers[4] == speakers[5] == "speaker_2" + assert None not in speakers + + +def test_multiple_supported_speakers_are_not_collapsed(): + speakers = cluster_speaker_embeddings( + [[1, 0, 0], [0.99, 0.05, 0], [0, 1, 0], [0.05, 0.99, 0], [0, 0, 1], [0, 0.05, 0.99]], + [segment(i * 2, i * 2 + 2) for i in range(6)], + ) + assert speakers == ["speaker_1", "speaker_1", "speaker_2", "speaker_2", "speaker_3", "speaker_3"] + + +def test_all_too_short_remains_unassigned_without_model_evidence(): + assert cluster_speaker_embeddings( + [None, None], [segment(0, 0.4), segment(0.5, 0.9)] + ) == [None, None] diff --git a/docs/development/多模态管线与模型运行开发说明.md b/docs/development/多模态管线与模型运行开发说明.md index 79eb184..fef05a1 100644 --- a/docs/development/多模态管线与模型运行开发说明.md +++ b/docs/development/多模态管线与模型运行开发说明.md @@ -53,7 +53,7 @@ API 保留 `backend/.venv`，模型依赖安装到独立的 `backend/.venv-model API 优先，无配置或无效结果时本地回退。local_only 禁止远程模型。纯文本附件和既有 sidecar 可导入，但已有真实音频时不使用旁边文本冒充识别。 -PyAV 提取音轨至 16 kHz 单声道，最长 1 小时，禁止解码器网络协议。能量分段后交给 Qwen3-ASR，返回片段边界，不宣称逐字对齐。ERes2NetV2 提取片段声纹并按相似度聚类；短片段、同段多人、重叠发言需要人工校对。缺失能力返回 DIARIZATION_UNAVAILABLE；未启用逐字对齐返回 WORD_TIMESTAMPS_UNAVAILABLE。 +PyAV 提取音轨至 16 kHz 单声道，最长 1 小时，禁止解码器网络协议。能量分段后交给 Qwen3-ASR，返回片段边界，不宣称逐字对齐。ERes2NetV2 提取片段声纹并按更新后的簇质心聚类；不足 1 秒的片段继承时间上最近的稳定说话人，3 秒以内且没有其他片段支持的离群簇并入最相近的稳定簇。由多个片段支持的少数说话人仍会保留。同段多人和重叠发言需要人工校对；仍无法分配的片段返回 DIARIZATION_PARTIAL。缺失能力返回 DIARIZATION_UNAVAILABLE；未启用逐字对齐返回 WORD_TIMESTAMPS_UNAVAILABLE。 术语是识别后的替换规则，保留原始文本和来源。重命名只修改显示名，稳定 ID 不变。笔记包含音频与时间跳转链接；重复导出不覆盖用户编辑。清理保留已导出笔记，音频链接失效，已清理任务不可重试。重建索引保留转写与笔记关联。 diff --git a/frontend/package.json b/frontend/package.json index b4c9c3a..52ac5da 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "notes-agent-frontend", "private": true, - "version": "0.3.2-alpha.1", + "version": "0.4.0-alpha.1", "type": "module", "scripts": { "dev": "vite", diff --git a/frontend/src-tauri/Cargo.lock b/frontend/src-tauri/Cargo.lock index 553e554..a31807c 100644 --- a/frontend/src-tauri/Cargo.lock +++ b/frontend/src-tauri/Cargo.lock @@ -3242,7 +3242,7 @@ dependencies = [ [[package]] name = "notesagent-desktop" -version = "0.3.2-alpha.1" +version = "0.4.0-alpha.1" dependencies = [ "argon2", "base64 0.22.1", diff --git a/frontend/src-tauri/Cargo.toml b/frontend/src-tauri/Cargo.toml index 0947675..4d1c9d3 100644 --- a/frontend/src-tauri/Cargo.toml +++ b/frontend/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "notesagent-desktop" -version = "0.3.2-alpha.1" +version = "0.4.0-alpha.1" edition = "2021" rust-version = "1.89" diff --git a/frontend/src-tauri/tauri.conf.json b/frontend/src-tauri/tauri.conf.json index b184527..1ad22c0 100644 --- a/frontend/src-tauri/tauri.conf.json +++ b/frontend/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "OpenNexus", - "version": "0.3.2-alpha.1", + "version": "0.4.0-alpha.1", "identifier": "cc.kronecker.notesagent", "build": { "beforeDevCommand": "pnpm dev", diff --git a/frontend/src/features/media/MediaView.vue b/frontend/src/features/media/MediaView.vue index eff0eb6..be188d0 100644 --- a/frontend/src/features/media/MediaView.vue +++ b/frontend/src/features/media/MediaView.vue @@ -58,6 +58,7 @@ const warningLabel = (warning: string) => warning.startsWith('MEDIA_CORRUPT_PACK ? t(`已跳过 ${warning.split(':')[1]} 个损坏音频包；缺失时长以静音保留，请校对受影响内容。`, `Skipped ${warning.split(':')[1]} damaged audio packets; missing duration was retained as silence. Review the affected content.`) : ({ DIARIZATION_UNAVAILABLE: t('当前无法分离说话人', 'Speaker identification is unavailable'), + DIARIZATION_PARTIAL: t('部分片段没有足够语音用于说话人识别，请人工校对', 'Some segments do not contain enough speech for speaker identification; review them manually'), WORD_TIMESTAMPS_UNAVAILABLE: t('未提供逐字时间戳', 'Word-level timestamps are unavailable'), DIARIZATION_SEGMENT_LEVEL: t('说话人按音频段估计，同段多人或重叠发言需人工校对', 'Speakers are estimated per segment; multiple or overlapping speakers require manual correction'), } as Record)[warning] || warning diff --git a/frontend/src/features/vault/VaultEntry.vue b/frontend/src/features/vault/VaultEntry.vue index 6733939..478325b 100644 --- a/frontend/src/features/vault/VaultEntry.vue +++ b/frontend/src/features/vault/VaultEntry.vue @@ -110,7 +110,7 @@ async function openFolderPicker() {
- v0.3.2-alpha.1 + v0.4.0-alpha.1