fix(editor): 支持完整 Shiki 语言并修复语言标识与图标展示

frontend/scripts/generate-language-icons.mjs

@@ -0,0 +1,45 @@
+// Usage: node scripts/generate-language-icons.mjs /path/to/@iconify-json/vscode-icons
+// Source: @iconify-json/vscode-icons 1.2.76 (MIT). No runtime network requests.
+import { readFileSync, writeFileSync } from 'node:fs'
+import { resolve } from 'node:path'
+import { bundledLanguagesInfo } from 'shiki/langs'
+
+const source = process.argv[2]
+if (!source) throw new Error('Provide the extracted vscode-icons package directory')
+const data = JSON.parse(readFileSync(resolve(source, 'icons.json'), 'utf8'))
+const overrides = {
+  ahk: 'autohotkey', ahk2: 'autohotkey', asm: 'assembly', bat: 'bat',
+  'angular-html': 'angular', 'angular-ts': 'angular',
+  'common-lisp': 'lisp', 'emacs-lisp': 'lisp',
+  'fortran-fixed-form': 'fortran', 'fortran-free-form': 'fortran',
+  'git-commit': 'git', 'git-rebase': 'git',
+  jsonc: 'json', jsonl: 'json', shellscript: 'shell', shellsession: 'shell',
+  jsx: 'reactjs', tsx: 'reactts', latex: 'tex', bibtex: 'bibtex',
+  'objective-c': 'objectivec', 'objective-cpp': 'objectivecpp',
+  dart: 'dartlang', d: 'dlang', v: 'vlang', gdshader: 'godot',
+  fish: 'shell', 'ssh-config': 'shell', 'vue-html': 'vue', 'vue-vine': 'vue',
+  'html-derivative': 'html', qss: 'qt', rbs: 'ruby',
+}
+const groups = new Map()
+const unmatched = []
+for (const info of [...bundledLanguagesInfo, { id: 'text', name: 'Text' }]) {
+  const candidates = [overrides[info.id], info.id, ...(info.aliases ?? []), info.name.toLowerCase().replace(/\s+/g, '')].filter(Boolean)
+  const icon = candidates.map(name => `file-type-${name}`).find(name => data.icons[name])
+  if (!icon) { unmatched.push(info.id); continue }
+  const ids = groups.get(icon) ?? []
+  ids.push(info.id)
+  groups.set(icon, ids)
+}
+const base = '.milkdown-host .language-list-item[data-language]'
+const svgUrl = icon => {
+  const item = data.icons[icon]
+  const svg = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ${item.width ?? data.width ?? 32} ${item.height ?? data.height ?? 32}">${item.body}</svg>`
+  return `url("data:image/svg+xml,${encodeURIComponent(svg)}")`
+}
+let css = `/* Generated by scripts/generate-language-icons.mjs. VSCode Icons (MIT); see language-icons-LICENSE.txt. */\n${base} { display: flex; align-items: center; gap: 8px; }\n${base}::before { content: ''; flex: 0 0 20px; width: 20px; height: 20px; background: center / contain no-repeat ${svgUrl('default-file')}; }\n`
+for (const [icon, ids] of groups) {
+  css += ids.map(id => `${base}[data-language="${id}"]::before`).join(',\n') + ` { background-image: ${svgUrl(icon)}; }\n`
+}
+writeFileSync(new URL('../src/features/editor/language-icons.css', import.meta.url), css)
+writeFileSync(new URL('../src/features/editor/language-icons-LICENSE.txt', import.meta.url), readFileSync(resolve(source, 'license.txt')))
+console.log(`${bundledLanguagesInfo.length + 1 - unmatched.length} languages mapped; generic file icon for: ${unmatched.join(', ')}`)

frontend/src/features/editor/VisualMarkdownEditor.spec.ts

@@ -3,13 +3,14 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'
 import { mount, type VueWrapper } from '@vue/test-utils'
 import { createPinia, setActivePinia } from 'pinia'
 import { editorViewCtx, type Editor } from '@milkdown/kit/core'
-import { TextSelection } from '@milkdown/kit/prose/state'
+import { NodeSelection, TextSelection } from '@milkdown/kit/prose/state'
 import { getMarkdown } from '@milkdown/kit/utils'
 import VisualMarkdownEditor from './VisualMarkdownEditor.vue'
 import { useSettingsStore } from '@/stores/settings'
 import { useThemeStore } from '@/stores/theme'
 import { codeBlockConfig } from '@milkdown/kit/component/code-block'
 import { EditorView as CodeMirror } from '@codemirror/view'
+import { renderMarkdown } from '@/utils/markdown'
 
 type EditorComponent = { getEditor: () => Editor | undefined }
 
@@ -48,6 +49,30 @@ afterEach(() => {
 })
 
 describe('VisualMarkdownEditor formatting toolbars', () => {
+  it.each([['jsonc', 'JSON with Comments', '// comment\n{"answer": 42}'], ['ahk', 'AutoHotkey', 'MsgBox "Hello"']])('persists %s from the language menu and renders it with Shiki', async (id, label, source) => {
+    const wrapper = mount(VisualMarkdownEditor, { props: { initialContent: `\`\`\`text\n${source}\n\`\`\`` }, attachTo: document.body })
+    mounted.push(wrapper)
+    const editor = await waitForEditor(wrapper)
+    editor.action(ctx => {
+      const view = ctx.get(editorViewCtx)
+      view.dispatch(view.state.tr.setSelection(NodeSelection.create(view.state.doc, 0)))
+    })
+    for (let attempt = 0; attempt < 100 && !wrapper.find('.language-button').exists(); attempt++) {
+      await new Promise(resolve => setTimeout(resolve, 10))
+    }
+    await wrapper.get('.language-button').trigger('click')
+    const item = wrapper.get(`.language-list-item[data-language="${id}"]`)
+    expect(item.text()).toBe(label)
+    await item.trigger('click')
+    const markdown = editor.action(getMarkdown())
+    expect(markdown).toContain(`\`\`\`${id}\n`)
+    const html = await renderMarkdown(markdown)
+    expect(new Set([...html.matchAll(/--shiki-light:([^;" ]+)/g)].map(match => match[1])).size).toBeGreaterThan(1)
+    const reopened = mount(VisualMarkdownEditor, { props: { initialContent: markdown }, attachTo: document.body })
+    mounted.push(reopened)
+    const restored = await waitForEditor(reopened)
+    expect(restored.action(ctx => ctx.get(editorViewCtx).state.doc.firstChild?.attrs.language)).toBe(id)
+  })
   it.each(['github-light', 'github-dark'] as const)('keeps Shiki %s mappings after Crepe merges its defaults', async theme => {
     useThemeStore().codeBlockTheme = theme
     const wrapper = mount(VisualMarkdownEditor, { props: { initialContent: '```python\nprint("Hello")\n```' }, attachTo: document.body })
@@ -57,10 +82,11 @@ describe('VisualMarkdownEditor formatting toolbars', () => {
     const matching = config.languages.filter(item => item.alias.includes('python'))
     expect(matching).toHaveLength(1)
     for (const name of ['Java', 'Go', 'Rust']) {
-      const language = config.languages.find(item => item.name === name)
+      const language = config.languages.find(item => item.name === name.toLowerCase())
       expect(language, `${name} remains available`).toBeDefined()
-      const support = await language!.load()
-      expect(support.language.parser.parse('class Example {}').length).toBe(16)
+      const view = new CodeMirror({ doc: 'class Example {}', extensions: [...config.extensions, await language!.load()] })
+      try { expect(view.dom.querySelector('.shiki-token')).not.toBeNull() }
+      finally { view.destroy() }
     }
     const cm = new CodeMirror({ doc: 'print("Hello")', extensions: [...config.extensions, await matching[0]!.load()] })
     try {

frontend/src/features/editor/VisualMarkdownEditor.vue

@@ -6,7 +6,8 @@ import { codeBlockConfig } from '@milkdown/kit/component/code-block'
 import { basicSetup } from 'codemirror'
 import { keymap } from '@codemirror/view'
 import { indentWithTab } from '@codemirror/commands'
-import { shikiEditorTheme, shikiLanguages } from './shikiCodeMirror'
+import { shikiEditorTheme, shikiLanguages, renderCodeLanguage } from './shikiCodeMirror'
+import './language-icons.css'
 import { installLanguagePickerPopover } from './languagePickerPopover'
 import {
   createCodeBlockCommand,
@@ -179,7 +180,8 @@ onMounted(async () => {
   // Replace both AFTER feature configuration to avoid default grammar collisions.
   crepe.editor.config(ctx => ctx.update(codeBlockConfig.key, config => ({
     ...config,
-    languages: shikiLanguages(themeStore.resolvedCodeBlockTheme, config.languages),
+    languages: shikiLanguages(themeStore.resolvedCodeBlockTheme),
+    renderLanguage: renderCodeLanguage,
     extensions: [basicSetup, keymap.of([indentWithTab]), shikiEditorTheme(themeStore.resolvedCodeBlockTheme)],
   })))
   crepe.editor.use(fontSizeMarkdownPlugin)

frontend/src/features/editor/language-icons-LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Roberto Huertas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

frontend/src/features/editor/shikiCodeMirror.spec.ts

@@ -1,7 +1,7 @@
 // @vitest-environment happy-dom
 import { afterEach, expect, it } from 'vitest'
 import { Compartment } from '@codemirror/state'
-import { LanguageDescription } from '@codemirror/language'
+import { bundledLanguagesInfo } from 'shiki/langs'
 import { EditorView } from '@codemirror/view'
 import { shikiLanguage, shikiLanguages } from './shikiCodeMirror'
 import { getCodeTokenizer } from '@/utils/markdown'
@@ -39,19 +39,29 @@ it('reconfigures language and theme without modifying the document', async () =>
 
 it('offers fenced-code aliases and retains the LaTeX selector', () => {
   const languages = shikiLanguages('github-light')
-  expect(languages.find(item => item.name === 'Python')?.alias).toContain('py')
-  expect(languages.find(item => item.name === 'LaTeX')?.alias).toContain('latex')
+  expect(languages.find(item => item.name === 'python')?.alias).toContain('py')
+  expect(languages.find(item => item.name === 'latex')?.alias).toContain('latex')
 })
 
-it('preserves original loaders and metadata while replacing supported languages', async () => {
-  const originalPython = LanguageDescription.of({ name: 'Python', alias: ['py', 'custom-python'], extensions: ['py'], filename: /^SConstruct$/, load: () => shikiLanguage('text', 'github-light') })
-  const originalRust = LanguageDescription.of({ name: 'Rust', alias: ['rs'], extensions: ['rs'], load: () => shikiLanguage('text', 'github-light') })
-  const languages = shikiLanguages('github-dark', [originalPython, originalRust])
-  expect(languages.find(item => item.name === 'Rust')).toBe(originalRust)
-  const python = languages.find(item => item.name === 'Python')!
-  expect(languages.filter(item => item.alias.includes('python'))).toHaveLength(1)
-  expect(python.alias).toContain('custom-python')
-  expect(python.extensions).toEqual(['py'])
-  expect(python.filename).toBe(originalPython.filename)
-  expect(await python.load()).not.toBe(await originalPython.load())
+it('offers every bundled Shiki language and alias', () => {
+  const languages = shikiLanguages('github-light')
+  expect(languages).toHaveLength(bundledLanguagesInfo.length + 1)
+  for (const info of bundledLanguagesInfo) {
+    const language = languages.find(item => item.alias.includes(info.id))!
+    expect(language, info.id).toBeDefined()
+    expect(language.name).toBe(info.id)
+    expect(language.alias).toContain(info.name.toLowerCase())
+    for (const alias of info.aliases ?? []) expect(language.alias).toContain(alias.toLowerCase())
+  }
 })
+
+it('loads every bundled grammar and produces tokens with both GitHub themes', async () => {
+  for (const info of bundledLanguagesInfo) {
+    for (const theme of ['github-light', 'github-dark'] as const) {
+      const tokenize = await getCodeTokenizer(theme, info.id)
+      const tokens = tokenize('example = 42', info.id).flat()
+      expect(tokens.map(token => token.content).join(''), info.id).toBe('example = 42')
+      expect(tokens.every(token => token.color), info.id).toBe(true)
+    }
+  }
+}, 120000)

frontend/src/features/editor/shikiCodeMirror.ts

@@ -1,11 +1,12 @@
 import { LanguageDescription, LanguageSupport, StreamLanguage } from '@codemirror/language'
 import { Decoration, EditorView, ViewPlugin, type DecorationSet, type ViewUpdate } from '@codemirror/view'
+import { bundledLanguagesInfo } from 'shiki/langs'
 import { getCodeTokenizer } from '@/utils/markdown'
 
 type CodeTheme = 'github-light' | 'github-dark'
 
 export async function shikiLanguage(language: string, theme: CodeTheme): Promise<LanguageSupport> {
-  const tokenize = await getCodeTokenizer(theme)
+  const tokenize = await getCodeTokenizer(theme, language)
   const highlights = ViewPlugin.fromClass(class {
     decorations: DecorationSet
 
@@ -39,42 +40,22 @@ export async function shikiLanguage(language: string, theme: CodeTheme): Promise
   return new LanguageSupport(parser, highlights)
 }
 
-export function shikiLanguages(theme: CodeTheme, originalLanguages: readonly LanguageDescription[] = []): LanguageDescription[] {
-  const overrides = [
-    { name: 'C', alias: ['c'] },
-    { name: 'C++', alias: ['cpp', 'c++'] },
-    { name: 'Python', alias: ['python', 'py'] },
-    { name: 'JavaScript', alias: ['javascript', 'js'] },
-    { name: 'TypeScript', alias: ['typescript', 'ts'] },
-    { name: 'HTML', alias: ['html'] },
-    { name: 'CSS', alias: ['css'] },
-    { name: 'JSON', alias: ['json'] },
-    { name: 'Shell', alias: ['shell', 'bash', 'sh'] },
-    { name: 'SQL', alias: ['sql'] },
-    { name: 'Markdown', alias: ['markdown', 'md'] },
-    { name: 'Plain text', alias: ['text', 'plaintext'] },
-    { name: 'LaTeX', alias: ['latex'] },
-  ].map(({ name, alias }) => LanguageDescription.of({
-    name, alias, load: () => shikiLanguage(alias[0]!, theme),
-  }))
-  // Keep Crepe's full registry and lazy loaders for languages without Shiki grammars.
-  const remaining = new Map(overrides.map(language => [language.name.toLowerCase(), language]))
-  const languages = originalLanguages.map(original => {
-    const key = original.name.toLowerCase()
-    const replacement = remaining.get(key)
-    if (!replacement) return original
-    remaining.delete(key)
-    return LanguageDescription.of({
-      name: original.name,
-      alias: [...new Set([...original.alias, ...replacement.alias])],
-      extensions: original.extensions,
-      filename: original.filename,
-      load: () => replacement.load(),
-    })
-  })
-  return [...languages, ...remaining.values()]
+export function shikiLanguages(theme: CodeTheme): LanguageDescription[] {
+  return [
+    ...bundledLanguagesInfo.map(info => LanguageDescription.of({
+      name: info.id,
+      alias: [info.name, ...(info.aliases ?? [])],
+      load: () => shikiLanguage(info.id, theme),
+    })),
+    LanguageDescription.of({ name: 'text', alias: ['Plain text', 'txt', 'plaintext'], load: () => shikiLanguage('text', theme) }),
+  ]
+}
+const languageLabels = new Map(bundledLanguagesInfo.flatMap(info =>
+  [info.id, info.name, ...(info.aliases ?? [])].map(alias => [alias.toLowerCase(), info.name] as const),
+))
+export function renderCodeLanguage(language: string): string {
+  return languageLabels.get(language.toLowerCase()) ?? (['text', 'txt', 'plaintext'].includes(language.toLowerCase()) ? 'Plain text' : language)
 }
-
 export function shikiEditorTheme(theme: CodeTheme) {
   return EditorView.theme({
     '&': { color: 'var(--color-code-text)', backgroundColor: 'var(--color-code-background)' },

frontend/src/utils/markdown.spec.ts

@@ -10,4 +10,12 @@ describe('Shiki GitHub 双主题', () => {
     expect(html).toContain('--shiki-light')
     expect(html).toContain('--shiki-dark')
   })
+
+  it('按需加载原先未支持的语言，并解析 Shiki 别名', async () => {
+    const source = 'fn main() { let answer = 42; }'
+    const [rust, alias] = await Promise.all([highlightCode(source, 'rust'), highlightCode(source, 'rs')])
+    expect(alias).toBe(rust)
+    expect(rust).toContain('github-dark')
+    expect(new Set([...rust.matchAll(/--shiki-light:([^;" ]+)/g)].map(match => match[1])).size).toBeGreaterThan(1)
+  })
 })

frontend/src/utils/markdown.ts

@@ -1,18 +1,8 @@
 import DOMPurify from 'dompurify'
 import { marked } from 'marked'
 import { createHighlighterCore } from 'shiki/core'
-import { createJavaScriptRegexEngine } from '@shikijs/engine-javascript'
-import css from '@shikijs/langs/css'
-import c from '@shikijs/langs/c'
-import cpp from '@shikijs/langs/cpp'
-import html from '@shikijs/langs/html'
-import javascript from '@shikijs/langs/javascript'
-import json from '@shikijs/langs/json'
-import markdown from '@shikijs/langs/markdown'
-import python from '@shikijs/langs/python'
-import shell from '@shikijs/langs/shellscript'
-import sql from '@shikijs/langs/sql'
-import typescript from '@shikijs/langs/typescript'
+import { createOnigurumaEngine } from 'shiki/engine/oniguruma'
+import { bundledLanguagesInfo } from 'shiki/langs'
 import githubDark from '@shikijs/themes/github-dark'
 import githubLight from '@shikijs/themes/github-light'
 
@@ -21,30 +11,46 @@ marked.setOptions({ gfm: true, breaks: true })
 // Highlighter 是昂贵的单例；复用初始化 Promise，避免每个代码块重复加载语法与主题。
 const highlighter = createHighlighterCore({
   themes: [githubLight, githubDark],
-  langs: [markdown, html, css, javascript, typescript, json, python, shell, sql, c, cpp],
-  engine: createJavaScriptRegexEngine(),
+  langs: [],
+  engine: createOnigurumaEngine(import('shiki/wasm')),
 })
 
-const languageAliases: Record<string, string> = {
-  bash: 'shell', js: 'javascript', md: 'markdown', plaintext: 'text', py: 'python', sh: 'shell', ts: 'typescript',
+const languageAliases = new Map(bundledLanguagesInfo.flatMap(info =>
+  [info.id, info.name, ...(info.aliases ?? [])].map(alias => [alias.toLowerCase(), info.id] as const),
+))
+const languageLoads = new Map<string, Promise<void>>()
+const languageLoaders = new Map(bundledLanguagesInfo.map(info => [info.id, info.import]))
+
+async function loadCodeLanguage(requestedLanguage: string) {
+  const shiki = await highlighter
+  const language = languageAliases.get(requestedLanguage.toLowerCase())
+  if (!language) return { shiki, language: 'text' as const }
+  let loading = languageLoads.get(language)
+  if (!loading) {
+    loading = shiki.loadLanguage(languageLoaders.get(language)!).catch(error => {
+      languageLoads.delete(language)
+      throw error
+    })
+    languageLoads.set(language, loading)
+  }
+  await loading
+  return { shiki, language }
 }
 
 export async function highlightCode(source: string, requestedLanguage = 'text'): Promise<string> {
-  const shiki = await highlighter
-  const language = languageAliases[requestedLanguage] ?? requestedLanguage
-  const loadedLanguage = shiki.getLoadedLanguages().includes(language as never) ? language : 'markdown'
+  const { shiki, language } = await loadCodeLanguage(requestedLanguage)
   return shiki.codeToHtml(source, {
-    lang: loadedLanguage,
+    lang: language,
     themes: { light: 'github-light', dark: 'github-dark' },
     defaultColor: false,
   })
 }
 
 /** Share the initialized grammar/theme registry with editable code blocks. */
-export async function getCodeTokenizer(theme: 'github-light' | 'github-dark') {
-  const shiki = await highlighter
+export async function getCodeTokenizer(theme: 'github-light' | 'github-dark', requestedLanguage = 'text') {
+  const { shiki } = await loadCodeLanguage(requestedLanguage)
   return (source: string, requestedLanguage: string) => {
-    const language = languageAliases[requestedLanguage] ?? requestedLanguage
+    const language = languageAliases.get(requestedLanguage.toLowerCase()) ?? 'text'
     return shiki.codeToTokens(source, {
       lang: shiki.getLoadedLanguages().includes(language as never) ? language : 'text',
       theme,