feat(desktop): add local voice dictation to the chat composer

Adds a microphone button beside the composer. Click to record, click to stop; the audio is resampled to 16 kHz mono PCM16 WAV, posted to the local server and transcribed by a SenseVoice worker process, and the text lands in the draft without being sent. If the draft changed or an IME is composing, the result is held behind an insert button instead of overwriting the user's text.

Server: a small provider registry behind /api/voice/* (catalog, preferences, prepare/cancel/status/remove, transcribe). The engine and model are downloaded at runtime to <config>/cc-haha/voice with pinned sha256/sha512, HuggingFace plus hf-mirror and npm plus npmmirror, HTTP Range resume, automatic retry after interruptions, and a partial file kept across cancels. Recognition runs in a separate worker process (sidecar --voice-worker), started on demand and reclaimed when idle.

Desktop: an independent Voice input settings tab with enable switch, model download progress and resume, language, microphone selection and a transcription test with a live waveform. Uses the shared Dropdown/Card/Button components; adds a danger-ghost Button variant. Preferences live in desktop-ui.json (schemaVersion 6); the microphone device id stays in localStorage.

Electron: main-window media permission handler limited to app pages, main frame and audio only, plus the audio-input entitlement and NSMicrophoneUsageDescription.

Scope: Electron desktop only. Not verified on Windows or Linux, with a real microphone, or in a signed and notarized package.
This commit is contained in:
程序员阿江(Relakkes)
2026-10-01 00:32:16 +08:00
parent 8d1548e62c
commit 6d75fac69e
94 changed files with 12839 additions and 20 deletions
@@ -11,6 +11,9 @@
<true/>
<key>com.apple.security.cs.disable-library-validation</key>
<true/>
<!-- Chromium's audio service runs in a helper process and captures the microphone. -->
<key>com.apple.security.device.audio-input</key>
<true/>
<key>com.apple.security.inherit</key>
<true/>
</dict>
+3
View File
@@ -15,5 +15,8 @@
and server runtime to work under a notarized hardened-runtime build. -->
<key>com.apple.security.cs.disable-library-validation</key>
<true/>
<!-- Voice dictation records from the microphone via getUserMedia in the renderer. -->
<key>com.apple.security.device.audio-input</key>
<true/>
</dict>
</plist>
+14 -1
View File
@@ -1,5 +1,5 @@
import { PublicAccessManager } from './services/publicAccess'
import { app, BrowserWindow, clipboard, dialog, ipcMain, Menu, nativeImage, nativeTheme, Notification, screen, session, WebContentsView } from 'electron'
import { app, BrowserWindow, clipboard, dialog, ipcMain, Menu, nativeImage, nativeTheme, Notification, screen, session, systemPreferences, WebContentsView } from 'electron'
import { autoUpdater } from 'electron-updater'
import path from 'node:path'
import { ELECTRON_EVENT_CHANNELS, ELECTRON_INTERNAL_CHANNELS, ELECTRON_IPC_CHANNELS, type ElectronIpcChannel } from './ipc/channels'
@@ -61,6 +61,7 @@ import {
startupWindowBackground,
type AppliedAppearance,
} from './services/nativeAppearance'
import { createRendererUrlMatcher, installMicrophonePermissions } from './services/microphonePermissions'
import { resolveRendererEntry } from './services/rendererEntry'
import { installRendererLifecycle } from './services/rendererLifecycle'
import { writeWindowSmokeSnapshot } from './services/windowSmoke'
@@ -899,6 +900,18 @@ async function createMainWindow() {
mainWindow!.webContents.id,
),
)
// Audio capture (dictation) is limited to application windows showing the app
// entry: the main window and the detached trace window, which renders the same
// chat input. Preview and workspace pages use their own partitions. Other
// permissions stay as before.
installMicrophonePermissions(mainWindow.webContents.session, {
isAppContents: contents => {
const owner = BrowserWindow.fromWebContents(contents)
return !!owner && !owner.isDestroyed()
},
isAppUrl: createRendererUrlMatcher(rendererEntry()),
systemPreferences,
})
installMainWindowNavigationGuards(mainWindow.webContents, { openExternal: openExternalUrl })
await installRendererContextMenu(mainWindow)
installPreviewCleanupOnRendererNavigation(mainWindow.webContents, () => {
@@ -0,0 +1,276 @@
import { existsSync, readFileSync } from 'node:fs'
import path from 'node:path'
import { pathToFileURL } from 'node:url'
import type { WebContents } from 'electron'
import { describe, expect, it, vi } from 'vitest'
import {
createRendererUrlMatcher,
installMicrophonePermissions,
type MediaAccessStatus,
} from './microphonePermissions'
const APP_ENTRY = '/Applications/Claude Code Haha.app/Contents/Resources/app.asar/dist/index.html'
const APP_URL = pathToFileURL(APP_ENTRY).toString()
type CheckHandler = (
contents: WebContents | null,
permission: string,
origin: string,
details: Record<string, unknown>,
) => boolean
type RequestHandler = (
contents: WebContents,
permission: string,
callback: (granted: boolean) => void,
details: Record<string, unknown>,
) => void
function setup(options: {
platform?: NodeJS.Platform
status?: MediaAccessStatus
ask?: () => Promise<boolean>
entry?: string
} = {}) {
const primary = { id: 1 } as unknown as WebContents
const trace = { id: 3 } as unknown as WebContents
const other = { id: 2 } as unknown as WebContents
const appContents = new Set<WebContents>([primary, trace])
let check!: CheckHandler
let request!: RequestHandler
const session = {
setPermissionCheckHandler: vi.fn((handler: CheckHandler) => { check = handler }),
setPermissionRequestHandler: vi.fn((handler: RequestHandler) => { request = handler }),
}
const systemPreferences = {
getMediaAccessStatus: vi.fn(() => options.status ?? 'granted'),
askForMediaAccess: vi.fn(options.ask ?? (async () => true)),
}
installMicrophonePermissions(session as never, {
isAppContents: contents => appContents.has(contents),
isAppUrl: createRendererUrlMatcher(options.entry ?? APP_ENTRY),
systemPreferences,
platform: options.platform ?? 'darwin',
})
const ask = (
contents: WebContents,
details: Record<string, unknown>,
permission = 'media',
) => new Promise<boolean>(resolve => {
request(contents, permission, resolve, {
isMainFrame: true,
requestingUrl: APP_URL,
mediaTypes: ['audio'],
...details,
})
})
const probe = (
contents: WebContents | null,
details: Record<string, unknown> = {},
permission = 'media',
) => check(contents, permission, 'file:///', {
isMainFrame: true,
requestingUrl: APP_URL,
mediaType: 'audio',
...details,
})
return {
primary,
trace,
other,
systemPreferences,
ask,
probe,
closeWindow: (contents: WebContents) => { appContents.delete(contents) },
}
}
describe('installMicrophonePermissions request handler', () => {
it('grants audio-only capture from the primary window main frame', async () => {
const { primary, ask } = setup({ platform: 'linux' })
await expect(ask(primary, {})).resolves.toBe(true)
})
it('grants audio-only capture to the detached trace window, whatever its query string', async () => {
const { trace, ask } = setup({ platform: 'linux' })
await expect(ask(trace, { requestingUrl: `${APP_URL}?traceWindow=1&traceSessionId=abc` })).resolves.toBe(true)
})
it('denies audio requests from contents that are not an application window', async () => {
const { other, ask, systemPreferences } = setup()
await expect(ask(other, {})).resolves.toBe(false)
expect(systemPreferences.askForMediaAccess).not.toHaveBeenCalled()
})
it('denies once the application window is gone', async () => {
const { primary, trace, ask, closeWindow } = setup({ platform: 'win32' })
closeWindow(primary)
closeWindow(trace)
await expect(ask(primary, {})).resolves.toBe(false)
await expect(ask(trace, {})).resolves.toBe(false)
})
it('still denies subframes and video from the trace window', async () => {
const { trace, ask } = setup({ platform: 'win32' })
await expect(ask(trace, { isMainFrame: false })).resolves.toBe(false)
await expect(ask(trace, { mediaTypes: ['audio', 'video'] })).resolves.toBe(false)
await expect(ask(trace, { requestingUrl: 'https://example.com/' })).resolves.toBe(false)
})
it('denies subframes and foreign or missing frame URLs', async () => {
const { primary, ask } = setup({ platform: 'win32' })
await expect(ask(primary, { isMainFrame: false })).resolves.toBe(false)
await expect(ask(primary, { requestingUrl: 'https://example.com/' })).resolves.toBe(false)
await expect(ask(primary, { requestingUrl: 'file:///tmp/other.html' })).resolves.toBe(false)
await expect(ask(primary, { requestingUrl: undefined })).resolves.toBe(false)
})
it('denies video, audio plus video, and requests without media types', async () => {
const { primary, ask } = setup({ platform: 'win32' })
await expect(ask(primary, { mediaTypes: ['video'] })).resolves.toBe(false)
await expect(ask(primary, { mediaTypes: ['audio', 'video'] })).resolves.toBe(false)
await expect(ask(primary, { mediaTypes: [] })).resolves.toBe(false)
await expect(ask(primary, { mediaTypes: undefined })).resolves.toBe(false)
})
it('leaves every other permission granted as Electron does by default', async () => {
const { primary, other, ask } = setup()
for (const permission of ['notifications', 'clipboard-read', 'clipboard-sanitized-write', 'fullscreen', 'openExternal', 'geolocation']) {
await expect(ask(primary, {}, permission)).resolves.toBe(true)
await expect(ask(other, { isMainFrame: false, requestingUrl: 'https://example.com/' }, permission)).resolves.toBe(true)
}
})
})
describe('installMicrophonePermissions on macOS', () => {
it('allows without prompting when the system already granted access', async () => {
const { primary, ask, systemPreferences } = setup({ status: 'granted' })
await expect(ask(primary, {})).resolves.toBe(true)
expect(systemPreferences.askForMediaAccess).not.toHaveBeenCalled()
})
it('asks the system when access is undetermined and follows the answer', async () => {
const granted = setup({ status: 'not-determined', ask: async () => true })
await expect(granted.ask(granted.primary, {})).resolves.toBe(true)
expect(granted.systemPreferences.askForMediaAccess).toHaveBeenCalledWith('microphone')
const refused = setup({ status: 'not-determined', ask: async () => false })
await expect(refused.ask(refused.primary, {})).resolves.toBe(false)
})
it('denies when the system prompt itself fails', async () => {
const { primary, ask } = setup({ status: 'not-determined', ask: async () => { throw new Error('boom') } })
await expect(ask(primary, {})).resolves.toBe(false)
})
it.each(['denied', 'restricted', 'unknown'] as const)('denies without prompting when the status is %s', async status => {
const { primary, ask, systemPreferences } = setup({ status })
await expect(ask(primary, {})).resolves.toBe(false)
expect(systemPreferences.askForMediaAccess).not.toHaveBeenCalled()
})
it('does not consult the system for non-media permissions', async () => {
const { primary, ask, systemPreferences } = setup({ status: 'denied' })
await expect(ask(primary, {}, 'notifications')).resolves.toBe(true)
expect(systemPreferences.getMediaAccessStatus).not.toHaveBeenCalled()
})
})
describe('installMicrophonePermissions check handler', () => {
it('reports audio as permitted for the primary main frame', () => {
const { primary, probe } = setup({ platform: 'win32' })
expect(probe(primary)).toBe(true)
})
it('requires the system grant on macOS and never prompts from a check', () => {
const granted = setup({ status: 'granted' })
expect(granted.probe(granted.primary)).toBe(true)
for (const status of ['not-determined', 'denied', 'restricted'] as const) {
const denied = setup({ status })
expect(denied.probe(denied.primary)).toBe(false)
expect(denied.systemPreferences.askForMediaAccess).not.toHaveBeenCalled()
}
})
it('reports audio as permitted for the trace window but not for non-application contents', () => {
const { trace, other, probe } = setup({ platform: 'win32' })
expect(probe(trace, { requestingUrl: `${APP_URL}?traceWindow=1` })).toBe(true)
expect(probe(other)).toBe(false)
})
it('denies other windows, null contents, subframes, foreign URLs and non-audio media', () => {
const { primary, other, probe } = setup({ platform: 'win32' })
expect(probe(other)).toBe(false)
expect(probe(null)).toBe(false)
expect(probe(primary, { isMainFrame: false })).toBe(false)
expect(probe(primary, { requestingUrl: 'https://example.com/' })).toBe(false)
expect(probe(primary, { requestingUrl: undefined })).toBe(false)
expect(probe(primary, { mediaType: 'video' })).toBe(false)
expect(probe(primary, { mediaType: 'unknown' })).toBe(false)
expect(probe(primary, { mediaType: undefined })).toBe(false)
})
it('keeps every other permission granted', () => {
const { other, probe } = setup({ status: 'denied' })
for (const permission of ['notifications', 'clipboard-read', 'fullscreen', 'openExternal']) {
expect(probe(other, { isMainFrame: false }, permission)).toBe(true)
}
})
})
describe('createRendererUrlMatcher', () => {
it('matches the packaged index.html regardless of query and hash', () => {
const matches = createRendererUrlMatcher(APP_ENTRY)
expect(matches(APP_URL)).toBe(true)
expect(matches(`${APP_URL}?traceWindow=1#/chat`)).toBe(true)
expect(matches(pathToFileURL('/Applications/Other.app/dist/index.html').toString())).toBe(false)
expect(matches('https://example.com/dist/index.html')).toBe(false)
expect(matches('not a url')).toBe(false)
})
it('matches only the dev server origin for an http entry', () => {
const matches = createRendererUrlMatcher('http://127.0.0.1:5173')
expect(matches('http://127.0.0.1:5173/?petWindow=1')).toBe(true)
expect(matches('http://127.0.0.1:5174/')).toBe(false)
expect(matches('http://localhost:5173/')).toBe(false)
expect(matches('file:///index.html')).toBe(false)
})
it('tolerates percent-encoding differences, and drive-letter case only on Windows', () => {
const entry = '/Applications/My App/dist/index.html'
const encoded = 'file:///Applications/My%20App/dist/index.html'
const differentCase = 'file:///applications/my%20app/dist/index.html'
expect(createRendererUrlMatcher(entry, 'darwin')(encoded)).toBe(true)
expect(createRendererUrlMatcher(entry, 'darwin')(differentCase)).toBe(false)
expect(createRendererUrlMatcher(entry, 'win32')(differentCase)).toBe(true)
})
})
describe('main window wiring', () => {
const desktopRoot = existsSync(path.resolve(process.cwd(), 'electron', 'main.ts'))
? process.cwd()
: path.resolve(process.cwd(), 'desktop')
const mainSource = readFileSync(path.join(desktopRoot, 'electron', 'main.ts'), 'utf8')
const mainWindowSource = mainSource.slice(
mainSource.indexOf('async function createMainWindow()'),
mainSource.indexOf('if (!acquireSingleInstanceLock'),
)
it('installs the microphone policy once, on the shared app session, for any application window', () => {
expect(mainWindowSource).toContain('installMicrophonePermissions(mainWindow.webContents.session')
expect(mainSource.match(/installMicrophonePermissions\(/g)).toHaveLength(1)
// A detached trace window renders the same ChatInput, so the gate cannot be "is mainWindow".
expect(mainWindowSource).toContain('BrowserWindow.fromWebContents(contents)')
expect(mainWindowSource).not.toMatch(/primary:/)
})
it('ships the macOS entitlement and usage description', () => {
for (const file of ['entitlements.mac.plist', 'entitlements.mac.inherit.plist']) {
const plist = readFileSync(path.join(desktopRoot, 'build', file), 'utf8')
expect(plist).toMatch(/<key>com\.apple\.security\.device\.audio-input<\/key>\s*<true\/>/)
}
const pkg = JSON.parse(readFileSync(path.join(desktopRoot, 'package.json'), 'utf8'))
expect(pkg.build.mac.extendInfo.NSMicrophoneUsageDescription).toMatch(/麦克风/)
expect(pkg.build.mac.extendInfo.NSMicrophoneUsageDescription).toMatch(/[Mm]icrophone/)
})
})
@@ -0,0 +1,121 @@
import { pathToFileURL } from 'node:url'
import type { Session, WebContents } from 'electron'
type PermissionSession = Pick<Session, 'setPermissionCheckHandler' | 'setPermissionRequestHandler'>
export type MediaAccessStatus = 'not-determined' | 'granted' | 'denied' | 'restricted' | 'unknown'
export type MicrophoneSystemPreferences = {
getMediaAccessStatus(mediaType: 'microphone'): MediaAccessStatus
askForMediaAccess(mediaType: 'microphone'): Promise<boolean>
}
export type MicrophonePermissionOptions = {
/**
* True for contents that live in a window this application created and that
* is still open (main window and the detached trace window, both of which
* render the chat input).
*/
isAppContents: (contents: WebContents) => boolean
/** True only for a frame URL that is the application's own renderer entry. */
isAppUrl: (url: string) => boolean
systemPreferences: MicrophoneSystemPreferences
platform?: NodeJS.Platform
}
/**
* Match the application's own renderer document: the dev server origin, or the
* packaged `dist/index.html`. Query and hash are ignored because the main and
* trace windows load the same entry with different query strings.
*/
export function createRendererUrlMatcher(
entry: string,
platform: NodeJS.Platform = process.platform,
): (url: string) => boolean {
if (/^https?:\/\//.test(entry)) {
const origin = new URL(entry).origin
return url => {
try {
return new URL(url).origin === origin
} catch {
return false
}
}
}
// Compare decoded paths (Chromium and pathToFileURL may encode differently)
// and, on Windows, case-insensitively, so drive-letter case cannot lock
// dictation out of an otherwise identical entry.
const normalize = (pathname: string) => {
let decoded = pathname
try {
decoded = decodeURIComponent(pathname)
} catch {
// Keep the raw path; it just will not match.
}
return platform === 'win32' ? decoded.toLowerCase() : decoded
}
const entryPath = normalize(pathToFileURL(entry).pathname)
return url => {
try {
const parsed = new URL(url)
return parsed.protocol === 'file:' && normalize(parsed.pathname) === entryPath
} catch {
return false
}
}
}
/**
* Grant audio-only capture to the main frame of application windows and leave every
* other permission at Electron's default. Installing any handler replaces that
* default, so the non-media branches explicitly keep the previous "granted"
* behaviour instead of silently tightening notifications, clipboard and so on.
*/
export function installMicrophonePermissions(
session: PermissionSession,
options: MicrophonePermissionOptions,
): void {
const platform = options.platform ?? process.platform
const prefs = options.systemPreferences
const fromApplication = (
contents: WebContents | null,
isMainFrame: boolean,
url: string | undefined,
) => {
if (!contents || !isMainFrame || !url) return false
return options.isAppContents(contents) && options.isAppUrl(url)
}
session.setPermissionCheckHandler((contents, permission, _origin, details) => {
if (permission !== 'media') return true
return fromApplication(contents, details.isMainFrame, details.requestingUrl)
&& details.mediaType === 'audio'
&& (platform !== 'darwin' || prefs.getMediaAccessStatus('microphone') === 'granted')
})
session.setPermissionRequestHandler((contents, permission, callback, details) => {
if (permission !== 'media') {
callback(true)
return
}
const mediaTypes = 'mediaTypes' in details ? details.mediaTypes : undefined
const audioOnly = mediaTypes?.length === 1 && mediaTypes[0] === 'audio'
if (!audioOnly || !fromApplication(contents, details.isMainFrame, details.requestingUrl)) {
callback(false)
return
}
if (platform !== 'darwin') {
callback(true)
return
}
const status = prefs.getMediaAccessStatus('microphone')
if (status === 'granted') {
callback(true)
} else if (status === 'not-determined') {
prefs.askForMediaAccess('microphone').then(callback, () => callback(false))
} else {
callback(false)
}
})
}
+15
View File
@@ -19,4 +19,19 @@ describe('desktop index startup diagnostics', () => {
expect(html).toContain('Startup resource failed to load:')
expect(html).toContain('Desktop app did not finish bootstrapping within')
})
it('lets the renderer play blob: audio, for the voice settings recording playback', () => {
const csp = html.match(/http-equiv="Content-Security-Policy"\s+content="([^"]+)"/)?.[1] ?? ''
const directives = new Map(csp.split(';').map(part => part.trim()).filter(Boolean).map((part) => {
const [name, ...sources] = part.split(/\s+/)
return [name, sources] as const
}))
// Without media-src, <audio src="blob:..."> falls back to default-src 'self' and is blocked.
expect(directives.get('media-src')).toEqual(["'self'", 'blob:'])
// Only media playback was opened up: the restrictive directives stay as they were.
expect(directives.get('default-src')).toEqual(["'self'"])
expect(directives.get('object-src')).toEqual(["'none'"])
expect(directives.get('worker-src')).toEqual(["'self'", 'blob:'])
})
})
+1 -1
View File
@@ -12,7 +12,7 @@
-->
<meta
http-equiv="Content-Security-Policy"
content="default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; font-src 'self' data:; img-src 'self' data: blob: https: http://127.0.0.1:* http://localhost:*; connect-src 'self' https: http://127.0.0.1:* http://localhost:* ws://127.0.0.1:* ws://localhost:*; worker-src 'self' blob:; object-src 'none'; base-uri 'self'"
content="default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; font-src 'self' data:; img-src 'self' data: blob: https: http://127.0.0.1:* http://localhost:*; connect-src 'self' https: http://127.0.0.1:* http://localhost:* ws://127.0.0.1:* ws://localhost:*; worker-src 'self' blob:; media-src 'self' blob:; object-src 'none'; base-uri 'self'"
/>
<!--
`viewport-fit=cover` MUST live here, in the markup. iOS WebKit reads the
+3
View File
@@ -48,6 +48,9 @@
"icon": "src-tauri/icons/icon.icns",
"hardenedRuntime": true,
"gatekeeperAssess": false,
"extendInfo": {
"NSMicrophoneUsageDescription": "仅在你点击听写按钮后使用麦克风录音,用于把语音转成文字。Microphone is used only after you click dictation, to turn speech into text."
},
"entitlements": "build/entitlements.mac.plist",
"entitlementsInherit": "build/entitlements.mac.inherit.plist",
"signIgnore": [
+10
View File
@@ -29,6 +29,16 @@ if (process.argv[2] === '--computer-use-repl-worker') {
process.exit(0)
}
// Local speech recognition runs in this same executable, relaunched by the
// server's voice recognizer. Like the worker above it must stay isolated: it
// reads its private configuration from the environment, loads the downloaded
// sherpa-onnx runtime by absolute path, and needs none of the desktop modes.
if (process.argv[2] === '--voice-worker') {
const { runVoiceWorker } = await import('../../src/server/services/voice/sensevoice/worker')
await runVoiceWorker()
process.exit(0)
}
type AdapterConfigShape = Awaited<
ReturnType<typeof import('../../adapters/common/config.ts')['loadConfig']>
>
+153
View File
@@ -0,0 +1,153 @@
// @vitest-environment node
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
import { mkdir, mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'
import { request } from 'node:http'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { describe, expect, it } from 'vitest'
import { makeWav } from '../../src/server/services/voice/sensevoice/__fixtures__/wav'
import { WORKER_CONFIG_ENV, WORKER_TOKEN_ENV } from '../../src/server/services/voice/sensevoice/protocol'
const repoRoot = path.resolve(import.meta.dirname, '../..')
const token = 'c'.repeat(64)
async function run(command: string, args: string[], cwd: string, env: NodeJS.ProcessEnv) {
const child = spawn(command, args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], timeout: 90_000 })
let stderr = ''
child.stderr.on('data', chunk => { stderr += String(chunk) })
const code = await new Promise<number | null>((resolve, reject) => {
child.once('error', reject)
child.once('exit', resolve)
})
if (code !== 0) throw new Error(`${path.basename(command)} exited ${code}: ${stderr}`)
}
function post(port: number, body: Uint8Array, headers: Record<string, string>) {
return new Promise<{ status: number; body: string }>((resolve, reject) => {
const req = request({
host: '127.0.0.1',
port,
method: 'POST',
path: '/transcribe?language=zh',
agent: false,
headers: { 'content-type': 'audio/wav', 'content-length': String(body.byteLength), ...headers },
}, response => {
let text = ''
response.on('data', chunk => { text += String(chunk) })
response.on('end', () => resolve({ status: response.statusCode ?? 0, body: text }))
})
req.once('error', reject)
req.end(Buffer.from(body))
})
}
describe.skipIf(process.platform === 'win32')('compiled desktop voice worker routing', () => {
it('boots the real merged entrypoint as a voice worker, loads the runtime by absolute path and serves the private socket', async () => {
const directory = await realpath(await mkdtemp(path.join(tmpdir(), 'cc-haha-sidecar-voice-worker-')))
const executable = path.join(directory, 'claude-sidecar-voice-fixture')
const env: NodeJS.ProcessEnv = {
PATH: process.env.PATH,
HOME: directory,
TMPDIR: directory,
CLAUDE_CONFIG_DIR: path.join(directory, '.claude'),
BUN_OPTIONS: '--no-env-file',
// preload.ts would chdir here. The worker must never load it.
CALLER_DIR: path.join(directory, 'must-not-enter-preload'),
}
let child: ChildProcessWithoutNullStreams | undefined
let exited: Promise<{ code: number | null; signal: NodeJS.Signals | null }> | undefined
try {
// Compile the real production entrypoint, not a handwritten worker shim.
const build = {
entrypoints: [path.join(repoRoot, 'desktop/sidecars/claude-sidecar.ts')],
features: ['TRANSCRIPT_CLASSIFIER'],
minify: { whitespace: true, identifiers: true, syntax: true },
sourcemap: 'none',
target: 'bun',
external: [
'@opentelemetry/exporter-trace-otlp-grpc', '@opentelemetry/exporter-trace-otlp-http',
'@opentelemetry/exporter-trace-otlp-proto', '@opentelemetry/exporter-logs-otlp-grpc',
'@opentelemetry/exporter-logs-otlp-http', '@opentelemetry/exporter-logs-otlp-proto',
'@opentelemetry/exporter-metrics-otlp-grpc', '@opentelemetry/exporter-metrics-otlp-http',
'@opentelemetry/exporter-metrics-otlp-proto', '@opentelemetry/exporter-prometheus',
'@aws-sdk/client-bedrock', '@aws-sdk/client-sts', '@anthropic-ai/bedrock-sdk',
'@anthropic-ai/foundry-sdk', '@anthropic-ai/vertex-sdk', '@azure/identity',
'@anthropic-ai/mcpb', 'fflate', 'sharp', 'react-devtools-core',
],
compile: { outfile: executable, autoloadTsconfig: true, autoloadPackageJson: true },
}
await run('bun', ['--no-env-file', '-e', `const r=await Bun.build(${JSON.stringify(build)});if(!r.success){console.error(r.logs);process.exit(1)}`], repoRoot, { ...env, CALLER_DIR: undefined })
if (process.platform === 'darwin') {
await run('/usr/bin/codesign', ['--remove-signature', executable], directory, env)
await run('/usr/bin/codesign', ['--sign', '-', '--force', '--timestamp=none', executable], directory, env)
}
// A stand-in for the downloaded runtime, at the layout the installer produces.
const runtimeRoot = path.join(directory, 'runtime')
const packageDir = path.join(runtimeRoot, 'node_modules', 'sherpa-onnx-node')
await mkdir(packageDir, { recursive: true })
await writeFile(path.join(packageDir, 'sherpa-onnx.js'), `
let language = 'auto'
class OfflineRecognizer {
constructor() {}
createStream() { return { samples: 0, acceptWaveform(a) { this.samples += a.samples.length } } }
setConfig(config) { language = config.modelConfig.senseVoice.language }
decode() {}
getResult(stream) { return { text: 'compiled-' + language + ':' + stream.samples } }
}
class Vad {
constructor() { this.total = 0; this.ready = [] }
acceptWaveform(samples) { this.total += samples.length }
isEmpty() { return this.ready.length === 0 }
front() { return { samples: new Float32Array(this.total) } }
pop() { this.ready.shift() }
reset() { this.total = 0; this.ready = [] }
flush() { this.ready.push(this.total) }
}
module.exports = { OfflineRecognizer, Vad }
`)
child = spawn(executable, ['--voice-worker'], {
cwd: directory,
stdio: 'pipe',
env: {
...env,
[WORKER_CONFIG_ENV]: JSON.stringify({
runtimeRoot, model: 'm', tokens: 't', vad: 'v', threads: 1,
maxAudioBytes: 4 * 1024 * 1024,
vadThreshold: 0.5, minSilenceSeconds: 0.5, minSpeechSeconds: 0.25, segmentSeconds: 30,
}),
[WORKER_TOKEN_ENV]: token,
},
})
exited = new Promise(resolve => child!.once('close', (code, signal) => resolve({ code, signal })))
let stderr = ''
child.stderr.on('data', chunk => { stderr += String(chunk) })
const port = await new Promise<number>((resolve, reject) => {
let text = ''
child!.stdout.on('data', chunk => {
text += String(chunk)
if (text.includes('\n')) resolve(JSON.parse(text.split('\n')[0]!).port)
})
child!.once('exit', code => reject(new Error(`voice worker exited ${code}: ${stderr}`)))
setTimeout(() => reject(new Error(`voice worker never became ready: ${stderr}`)), 20_000)
})
const wav = makeWav(1)
const ok = await post(port, wav, { authorization: `Bearer ${token}` })
expect(ok.status).toBe(200)
expect(JSON.parse(ok.body)).toMatchObject({ text: 'compiled-zh:16000', audioSeconds: 1 })
expect((await post(port, wav, { authorization: 'Bearer nope' })).status).toBe(401)
// The parent going away (stdin closing) is what ends the worker.
child.stdin.end()
expect(await exited).toEqual({ code: 0, signal: null })
expect(stderr).toBe('')
} finally {
child?.kill('SIGKILL')
if (exited) await exited
await rm(directory, { recursive: true, force: true })
}
}, 120_000)
})
+3
View File
@@ -1,4 +1,5 @@
import { ApiError, api, getApiUrl, getAuthToken } from './client'
import type { VoicePreferences } from './voice'
export type SidebarProjectPreferences = {
projectOrder: string[]
@@ -30,6 +31,8 @@ export type DesktopUiPreferences = {
sidebar: SidebarProjectPreferences
profile: DesktopProfilePreferences
pet: DesktopPetPreferences
/** Absent when talking to a server older than the voice input feature. */
voiceInput?: VoicePreferences
projectDisplayNames: Record<string, string>
}
+67
View File
@@ -0,0 +1,67 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { ApiError, getDefaultBaseUrl, setAuthToken, setBaseUrl } from './client'
import { voiceApi } from './voice'
const wav = new Blob([new Uint8Array(48)], { type: 'audio/wav' })
const options = { providerId: 'sensevoice-local', language: 'zh' } as const
async function caught(promise: Promise<unknown>): Promise<ApiError> {
try {
await promise
} catch (error) {
expect(error).toBeInstanceOf(ApiError)
return error as ApiError
}
throw new Error('expected transcribe to reject')
}
describe('voiceApi.transcribe', () => {
afterEach(() => {
setAuthToken(null)
setBaseUrl(getDefaultBaseUrl())
vi.restoreAllMocks()
})
it('posts the WAV as the raw body with the provider and language in the query', async () => {
setBaseUrl('http://127.0.0.1:49237')
setAuthToken('token-1')
const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue(
Response.json({ text: 'hello', audioSeconds: 1, inferenceSeconds: 0.1 }),
)
await expect(voiceApi.transcribe(wav, options)).resolves.toEqual({
text: 'hello',
audioSeconds: 1,
inferenceSeconds: 0.1,
})
const [url, init] = fetchMock.mock.calls[0]!
expect(String(url)).toBe('http://127.0.0.1:49237/api/voice/transcribe?provider=sensevoice-local&language=zh')
expect(init).toMatchObject({ method: 'POST', body: wav })
expect(init!.headers).toMatchObject({ 'Content-Type': 'audio/wav', Authorization: 'Bearer token-1' })
})
it('keeps the error code from a JSON error body', async () => {
setBaseUrl('http://127.0.0.1:49237')
vi.spyOn(globalThis, 'fetch').mockResolvedValue(
Response.json({ error: 'voice/not-ready', message: 'not ready' }, { status: 409 }),
)
const error = await caught(voiceApi.transcribe(wav, options))
expect(error.status).toBe(409)
expect(error.body).toEqual({ error: 'voice/not-ready', message: 'not ready' })
})
it('keeps the status and raw text when the error body is not JSON', async () => {
setBaseUrl('http://127.0.0.1:49237')
vi.spyOn(globalThis, 'fetch').mockResolvedValue(
new Response('<html>Bad Gateway</html>', { status: 502, headers: { 'Content-Type': 'text/html' } }),
)
const error = await caught(voiceApi.transcribe(wav, options))
expect(error.status).toBe(502)
expect(error.body).toBe('<html>Bad Gateway</html>')
})
})
+128
View File
@@ -0,0 +1,128 @@
import { ApiError, api, getApiUrl, getAuthToken } from './client'
/**
* Wire types mirror `src/server/services/voice/types.ts`. Keep them in sync;
* the server's `contractParity.test.ts` pins them against each other.
*/
export type VoiceLanguage = 'auto' | 'zh' | 'en' | 'ja' | 'ko' | 'yue'
export type VoicePreparationPhase = 'unprepared' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'cancelled'
export type VoicePreparationStep = 'runtime' | 'model' | 'vad' | 'verify'
export type VoiceFailureReason =
| 'network'
| 'dns'
| 'timeout'
| 'certificate'
| 'http'
| 'integrity'
| 'storage'
| 'unsupported-platform'
| 'unknown'
export type VoiceFailure = {
reason: VoiceFailureReason
source?: string
status?: number
resource?: string
message: string
}
export type VoicePreparationState = {
phase: VoicePreparationPhase
step?: VoicePreparationStep
resource?: string
completedBytes?: number
totalBytes?: number
resumedFromBytes?: number
source?: string
error?: VoiceFailure
}
export type VoiceProviderInfo = {
id: string
name: string
location: 'local' | 'cloud'
languages: VoiceLanguage[]
downloadBytes?: number
}
export type VoiceProviderStatus = {
info: VoiceProviderInfo
preparation: VoicePreparationState
}
export type VoicePreferences = {
enabled: boolean
providerId: string
language: VoiceLanguage
}
export type VoiceLimits = {
maxAudioSeconds: number
maxAudioBytes: number
}
export type VoiceCatalog = {
supported: boolean
providers: VoiceProviderStatus[]
preferences: VoicePreferences
limits: VoiceLimits
}
export type VoiceTranscript = {
text: string
audioSeconds: number
inferenceSeconds: number
}
export type VoiceErrorCode = 'voice/invalid-audio' | 'voice/not-ready' | 'voice/failed' | 'voice/unknown-provider'
export const voiceApi = {
catalog: () => api.get<VoiceCatalog>('/api/voice/catalog'),
updatePreferences: (patch: Partial<VoicePreferences>) =>
api.put<{ preferences: VoicePreferences }>('/api/voice/preferences', patch),
/** Starts (or joins) the download for a provider; poll `providerStatus` for progress. */
prepare: (providerId: string) =>
api.post<VoiceProviderStatus>(`/api/voice/providers/${encodeURIComponent(providerId)}/prepare`),
cancelPrepare: (providerId: string) =>
api.post<VoiceProviderStatus>(`/api/voice/providers/${encodeURIComponent(providerId)}/cancel`),
providerStatus: (providerId: string) =>
api.get<VoiceProviderStatus>(`/api/voice/providers/${encodeURIComponent(providerId)}/status`),
/** Deletes downloaded runtime and models for a provider. */
removeAssets: (providerId: string) =>
api.delete<VoiceProviderStatus>(`/api/voice/providers/${encodeURIComponent(providerId)}/assets`),
/** Uploads a 16 kHz mono PCM16 WAV as the raw request body. */
async transcribe(
wav: Blob,
options: { providerId: string; language: VoiceLanguage; signal?: AbortSignal },
): Promise<VoiceTranscript> {
const headers: Record<string, string> = { 'Content-Type': 'audio/wav' }
const token = getAuthToken()
if (token) headers.Authorization = `Bearer ${token}`
const query = new URLSearchParams({ provider: options.providerId, language: options.language })
const res = await fetch(getApiUrl(`/api/voice/transcribe?${query}`), {
method: 'POST',
headers,
body: wav,
signal: options.signal,
})
if (!res.ok) {
// Read the body once: a proxy error page is not JSON, and a second read would throw.
const text = await res.text().catch(() => '')
let body: unknown = text
try {
body = JSON.parse(text)
} catch {
// Keep the raw text.
}
throw new ApiError(res.status, body)
}
return res.json() as Promise<VoiceTranscript>
},
}
+154 -1
View File
@@ -35,6 +35,15 @@ const mocks = vi.hoisted(() => ({
dialogOpen: vi.fn(),
webviewDragHandlers: [] as Array<(event: { payload: unknown }) => void>,
webviewUnlisten: vi.fn(),
voiceSupported: vi.fn(() => false),
voiceStartRecording: vi.fn(),
voiceTranscribe: vi.fn(),
voiceCatalog: vi.fn(async () => ({
supported: false,
providers: [],
preferences: { enabled: false, providerId: 'sensevoice-local', language: 'auto' },
limits: { maxAudioSeconds: 60, maxAudioBytes: 1_000_000 },
})),
}))
vi.mock('@/lib/workspace/openSideChat', () => ({ openSideChat: mocks.sideOpen }))
@@ -59,6 +68,24 @@ vi.mock('../../api/composerReferences', () => ({
composerReferencesApi: { list: mocks.listReferences },
}))
vi.mock('@/api/voice', () => ({
voiceApi: {
catalog: mocks.voiceCatalog,
transcribe: mocks.voiceTranscribe,
providerStatus: vi.fn(),
updatePreferences: vi.fn(),
prepare: vi.fn(),
cancelPrepare: vi.fn(),
removeAssets: vi.fn(),
},
}))
vi.mock('@/features/voiceInput/recorder', async (importOriginal) => ({
...(await importOriginal<typeof import('@/features/voiceInput/recorder')>()),
isVoiceCaptureSupported: mocks.voiceSupported,
startRecording: mocks.voiceStartRecording,
}))
vi.mock('../../api/agents', () => ({
agentsApi: {
list: mocks.listAgents,
@@ -118,7 +145,8 @@ vi.mock('../controls/ModelSelector', async () => {
})
import { ChatInput } from './ChatInput'
import { getComposerElement, getComposerText, setComposerText } from './composerTestUtils'
import { getComposerElement, getComposerText, getComposerView, setComposerSelection, setComposerText } from './composerTestUtils'
import { useVoiceInputStore } from '../../stores/voiceInputStore'
import { useChatStore } from '../../stores/chatStore'
import { useSessionStore } from '../../stores/sessionStore'
import { useSettingsStore } from '../../stores/settingsStore'
@@ -218,6 +246,8 @@ describe('ChatInput file mentions', () => {
vi.clearAllMocks()
useSideChatStore.setState({ entries: {} })
useTeamPlanStore.setState({ bySession: {} })
mocks.voiceSupported.mockReturnValue(false)
useVoiceInputStore.setState({ catalog: null, loading: false, error: null })
mocks.sideOpen.mockResolvedValue('side-tab')
mocks.createRepositoryBranch.mockReset()
act(() => {
@@ -3160,4 +3190,127 @@ describe('ChatInput file mentions', () => {
updateUser.mockRestore()
}
})
describe('voice input', () => {
let finishTranscription: (text: string) => void
const recording = () => ({
getLevel: vi.fn(() => 0),
stop: vi.fn(async () => ({ wav: new Blob(['wav']), seconds: 2 })),
cancel: vi.fn(),
})
function armVoice() {
mocks.voiceSupported.mockReturnValue(true)
useVoiceInputStore.setState({
catalog: {
supported: true,
providers: [{
info: { id: 'sensevoice-local', name: 'SenseVoice', location: 'local', languages: ['auto', 'zh'] },
preparation: { phase: 'ready' },
}],
preferences: { enabled: true, providerId: 'sensevoice-local', language: 'zh' },
limits: { maxAudioSeconds: 60, maxAudioBytes: 1_000_000 },
},
})
mocks.voiceStartRecording.mockImplementation(async () => activeRecording)
mocks.voiceTranscribe.mockImplementation(() => new Promise((resolve) => {
finishTranscription = (text) => resolve({ text, audioSeconds: 2, inferenceSeconds: 0.1 })
}))
}
let activeRecording: ReturnType<typeof recording>
async function dictate() {
await act(async () => {
fireEvent.click(screen.getByRole('button', { name: 'Dictate' }))
})
await act(async () => {
fireEvent.click(await screen.findByRole('button', { name: 'Stop recording and transcribe' }))
})
await screen.findByRole('button', { name: 'Transcribing…' })
}
beforeEach(() => {
activeRecording = recording()
armVoice()
})
it('puts the microphone between the model picker and the send button', () => {
render(<ChatInput />)
const trailing = screen.getByTestId('chat-input-toolbar-trailing')
const order = Array.from(trailing.children).map((child) => (child as HTMLElement).dataset.testid ?? child.textContent)
const model = order.indexOf('model-selector-shell')
const voice = order.indexOf('voice-input')
expect(model).toBeGreaterThanOrEqual(0)
expect(voice).toBe(model + 1)
expect(trailing.lastElementChild).toBe(screen.getByRole('button', { name: 'Run' }))
})
it('does not render the microphone until dictation is ready', () => {
useVoiceInputStore.setState({
catalog: { ...useVoiceInputStore.getState().catalog!, preferences: { enabled: false, providerId: 'sensevoice-local', language: 'zh' } },
})
render(<ChatInput />)
expect(screen.queryByTestId('voice-input')).toBeNull()
})
it('writes dictated text at the caret without sending anything', async () => {
render(<ChatInput />)
setComposerText('ab', 1)
vi.spyOn(getComposerView(), 'hasFocus').mockReturnValue(true)
setComposerSelection(1)
await dictate()
await act(async () => {
finishTranscription('你好')
})
expect(getComposerText()).toBe('a你好b')
expect(mocks.wsSend).not.toHaveBeenCalled()
})
it('keeps the text aside when the message was sent while it was being recognised', async () => {
render(<ChatInput />)
setComposerText('question')
await dictate()
fireEvent.click(screen.getByRole('button', { name: 'Run' }))
expect(getComposerText()).toBe('')
await act(async () => {
finishTranscription('late words')
})
// The draft looked identical (empty) before and after, but it is a
// different draft; the text waits for the user instead of landing in it.
expect(getComposerText()).toBe('')
expect(screen.getByTestId('voice-input-pending-text')).toHaveTextContent('late words')
})
it('holds a result that arrives during an IME composition in the composer', async () => {
render(<ChatInput />)
await dictate()
fireEvent.compositionStart(getComposerElement())
await act(async () => {
finishTranscription('你好')
})
expect(getComposerText()).toBe('')
expect(screen.getByTestId('voice-input-pending-text')).toHaveTextContent('你好')
})
it('abandons the recording when the composer is hidden', async () => {
const { rerender } = render(<ChatInput sessionId={sessionId} />)
await act(async () => {
fireEvent.click(screen.getByRole('button', { name: 'Dictate' }))
})
await screen.findByRole('button', { name: 'Stop recording and transcribe' })
rerender(<ChatInput sessionId={sessionId} visible={false} />)
expect(activeRecording.cancel).toHaveBeenCalledTimes(1)
})
})
})
+15 -4
View File
@@ -75,6 +75,8 @@ import {
import type { PermissionMode } from '../../types/settings'
import { getSessionWorkspaceState, getSessionSeedWorkDir } from '../../lib/sessionWorkspace'
import { hasRunningSubagentTasks } from '../../lib/backgroundTasks'
import { useComposerDictation } from '@/features/voiceInput/useComposerDictation'
import { VoiceInputButton } from '@/features/voiceInput/VoiceInputButton'
type GitInfo = SessionGitInfo
@@ -307,6 +309,14 @@ export function ChatInput({ variant = 'default', compact = false, sessionId, vis
// they used to spell the condition out separately, which is how one branch
// ends up locked while the other keeps accepting text.
const composerDisabled = isWorkspaceMissing || launchTransitioning || isPreparingTurn || questionPending
// A hidden composer keeps its state but is not where the user is looking, so
// dictation started in it must not survive the switch.
const dictation = useComposerDictation({
composerRef,
draft: input,
blocked: composerDisabled,
contextKey: visible ? activeTabId : null,
})
const hasWorkspaceReferences = !isMemberSession && workspaceReferences.length > 0
const isHeroComposer = variant === 'hero' && !isMemberSession && !compact
const resolvedWorkDir = activeSession?.workDir || gitInfo?.workDir || undefined
@@ -1467,8 +1477,8 @@ export function ChatInput({ variant = 'default', compact = false, sessionId, vis
onChange={handleComposerChange}
onKeyDown={handleComposerKeyDown}
onPaste={handleComposerPaste}
onCompositionStart={() => { composingRef.current = true }}
onCompositionEnd={() => { composingRef.current = false }}
onCompositionStart={() => { composingRef.current = true; dictation.compositionHandlers.onCompositionStart() }}
onCompositionEnd={() => { composingRef.current = false; dictation.compositionHandlers.onCompositionEnd() }}
placeholder={composerPlaceholder}
disabled={composerDisabled}
// `min-w-0`: a paragraph holding an unbreakable run (a long URL,
@@ -1501,8 +1511,8 @@ export function ChatInput({ variant = 'default', compact = false, sessionId, vis
onChange={handleComposerChange}
onKeyDown={handleComposerKeyDown}
onPaste={handleComposerPaste}
onCompositionStart={() => { composingRef.current = true }}
onCompositionEnd={() => { composingRef.current = false }}
onCompositionStart={() => { composingRef.current = true; dictation.compositionHandlers.onCompositionStart() }}
onCompositionEnd={() => { composingRef.current = false; dictation.compositionHandlers.onCompositionEnd() }}
placeholder={composerPlaceholder}
disabled={composerDisabled}
editorClassName={`chat-reading-text max-h-[200px] overflow-y-auto text-sm leading-relaxed text-[var(--color-text-primary)] ${
@@ -1657,6 +1667,7 @@ export function ChatInput({ variant = 'default', compact = false, sessionId, vis
fluid
/>
)}
<VoiceInputButton dictation={dictation} blocked={composerDisabled} mobile={isMobileComposer} />
{!isMemberSession && !isActive && (hasRunningSubagents || hasRunningTeam) ? (
<Button
variant="danger"
@@ -42,6 +42,14 @@ export type MentionComposerHandle = {
/** Caret/selection as offsets in the projected plain text. */
getSelectionOffsets: () => { start: number; end: number }
setSelectionOffsets: (start: number, end?: number) => void
/** Whether the editable element currently holds keyboard focus. */
hasFocus: () => boolean
/**
* Replaces the projected-text range with plain text as one editor
* transaction, so a single undo removes it (unlike a `value` rewrite, which
* resets history). The caret lands after the inserted text.
*/
insertTextAtOffsets: (start: number, end: number, text: string) => void
/**
* Content for the model: text with each mention pill serialized to
* file paths or explicit skill/plugin requests. Read from the live document, so literal text that
@@ -342,6 +350,17 @@ export const MentionComposer = forwardRef<MentionComposerHandle, MentionComposer
: TextSelection.create(view.state.doc, from, to)
view.dispatch(view.state.tr.setSelection(selection))
},
hasFocus: () => viewRef.current?.hasFocus() ?? false,
insertTextAtOffsets: (start, end, text) => {
const view = viewRef.current
if (!view || !text) return
const docLength = projectedDocLength(view.state.doc)
const from = textOffsetToPmPos(view.state.doc, Math.min(start, docLength))
const to = textOffsetToPmPos(view.state.doc, Math.min(Math.max(start, end), docLength))
const tr = view.state.tr.insertText(text, from, to)
tr.setSelection(TextSelection.near(tr.doc.resolve(from + text.length)))
view.dispatch(tr.scrollIntoView())
},
getModelContent: () => {
const view = viewRef.current
return view ? serializeComposerDoc(view.state.doc) : ''
+1 -1
View File
@@ -5,7 +5,7 @@ import { describe, expect, it, vi } from 'vitest'
import { Button, type ButtonSize, type ButtonVariant } from './Button'
const VARIANTS: ButtonVariant[] = ['primary', 'secondary', 'tonal', 'ghost', 'danger', 'danger-outline', 'link', 'inverse']
const VARIANTS: ButtonVariant[] = ['primary', 'secondary', 'tonal', 'ghost', 'danger', 'danger-outline', 'danger-ghost', 'link', 'inverse']
const SIZES: ButtonSize[] = ['xs', 'sm', 'base', 'md', 'lg']
describe('Button', () => {
+8
View File
@@ -22,6 +22,12 @@ export type ButtonVariant =
| 'ghost'
| 'danger'
| 'danger-outline'
/**
* Danger-colored text with no border or fill until hovered, for a destructive
* action that is rarely wanted and should not compete with the page's real
* primary action (unlike `danger-outline`, which reads as a call to act).
*/
| 'danger-ghost'
| 'link'
/**
* Inverted fill — dark on light themes, light on dark. For a neutral but
@@ -110,6 +116,8 @@ const VARIANT_CLASSES: Record<ButtonVariant, string> = {
`bg-[var(--color-inverse-surface)] text-[var(--color-inverse-on-surface)] hover:bg-[var(--color-brand)] hover:-translate-y-px active:translate-y-0 active:scale-[0.97] ${DISABLED_FILL}`,
'danger-outline':
`bg-transparent text-[var(--color-error)] border border-[var(--color-error)] hover:bg-[var(--color-error-soft)] active:scale-[0.98] ${DISABLED_FADE}`,
'danger-ghost':
`bg-transparent text-[var(--color-error)] hover:bg-[var(--color-error-soft)] active:scale-[0.98] ${DISABLED_FADE}`,
link:
`bg-transparent text-[var(--color-brand)] underline-offset-2 hover:underline hover:text-[var(--color-brand-hover)] px-0 ${DISABLED_FADE}`,
}
+1 -1
View File
@@ -54,7 +54,7 @@ import type {
/** Sourced from the type rather than restated, so a new palette shows up here. */
const THEMES = THEME_MODES
const TONES: Tone[] = ['neutral', 'brand', 'success', 'warning', 'danger', 'info']
const VARIANTS: ButtonVariant[] = ['primary', 'secondary', 'tonal', 'tonal-outline', 'ghost', 'danger', 'danger-outline', 'link', 'inverse']
const VARIANTS: ButtonVariant[] = ['primary', 'secondary', 'tonal', 'tonal-outline', 'ghost', 'danger', 'danger-outline', 'danger-ghost', 'link', 'inverse']
const SIZES: ButtonSize[] = ['xs', 'sm', 'base', 'md', 'lg']
const ICON_SIZES: IconButtonSize[] = ['2xs', 'xs', 'sm', 'md', 'lg', 'xl', '2xl']
const ICON_TONES: IconButtonTone[] = ['default', 'secondary', 'muted', 'brand', 'danger']
@@ -0,0 +1,735 @@
import { useEffect, useRef, useState } from 'react'
import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react'
import { undo } from 'prosemirror-history'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import '@testing-library/jest-dom'
import { ApiError } from '@/api/client'
import type { VoiceCatalog } from '@/api/voice'
import { getComposerViewForTesting, MentionComposer, type MentionComposerHandle } from '@/components/chat/MentionComposer'
import { translate } from '@/i18n'
import type { TranslationKey } from '@/i18n/locales/en'
import { useSettingsStore } from '@/stores/settingsStore'
import { selectVoiceInputReady, useVoiceInputStore } from '@/stores/voiceInputStore'
import { VoiceRecorderError, type StartRecordingOptions } from './recorder'
import { useComposerDictation } from './useComposerDictation'
import { VoiceInputButton } from './VoiceInputButton'
const mocks = vi.hoisted(() => ({
transcribe: vi.fn(),
catalog: vi.fn(),
startRecording: vi.fn(),
supported: vi.fn(() => true),
}))
vi.mock('@/api/voice', () => ({
voiceApi: {
transcribe: mocks.transcribe,
catalog: mocks.catalog,
providerStatus: vi.fn(),
updatePreferences: vi.fn(),
prepare: vi.fn(),
cancelPrepare: vi.fn(),
removeAssets: vi.fn(),
},
}))
vi.mock('./recorder', async (importOriginal) => ({
...(await importOriginal<typeof import('./recorder')>()),
startRecording: mocks.startRecording,
isVoiceCaptureSupported: mocks.supported,
}))
const en = (key: TranslationKey) => translate('en', key)
function catalogFixture(overrides: Partial<VoiceCatalog['preferences']> = {}, phase: 'ready' | 'unprepared' = 'ready'): VoiceCatalog {
return {
supported: true,
providers: [{
info: { id: 'sensevoice-local', name: 'SenseVoice', location: 'local', languages: ['auto', 'zh', 'en'] },
preparation: { phase },
}],
preferences: { enabled: true, providerId: 'sensevoice-local', language: 'zh', ...overrides },
limits: { maxAudioSeconds: 60, maxAudioBytes: 10_000_000 },
}
}
type FakeRecording = {
getLevel: ReturnType<typeof vi.fn>
stop: ReturnType<typeof vi.fn>
cancel: ReturnType<typeof vi.fn>
}
function fakeRecording(seconds = 2): FakeRecording {
return {
getLevel: vi.fn(() => 0.4),
stop: vi.fn(async () => ({ wav: new Blob(['wav']), seconds })),
cancel: vi.fn(),
}
}
let recording: FakeRecording
let startOptions: StartRecordingOptions
let finishTranscription: (text: string) => void
let failTranscription: (error: unknown) => void
let transcribeSignal: AbortSignal | undefined
let handle: MentionComposerHandle | null
type HarnessProps = { initial?: string; blocked?: boolean; contextKey?: string | null }
function Harness({ initial = '', blocked = false, contextKey = 'session-a' }: HarnessProps) {
const [input, setInput] = useState(initial)
const [mentions, setMentions] = useState<never[]>([])
const ref = useRef<MentionComposerHandle>(null)
const dictation = useComposerDictation({ composerRef: ref, draft: input, blocked, contextKey })
useEffect(() => {
handle = ref.current
})
return (
<div>
<MentionComposer
ref={ref}
value={input}
mentions={mentions}
onChange={(text) => { setInput(text); setMentions([]) }}
onCompositionStart={dictation.compositionHandlers.onCompositionStart}
onCompositionEnd={dictation.compositionHandlers.onCompositionEnd}
/>
<VoiceInputButton dictation={dictation} blocked={blocked} />
<output data-testid="draft">{input}</output>
</div>
)
}
function editorView() {
const editor = document.querySelector<HTMLElement>('[data-composer-editor]')
const view = getComposerViewForTesting(editor)
if (!editor || !view) throw new Error('composer not mounted')
return { editor, view }
}
/** A user edit: goes through the editor exactly like typing does. */
function typeText(text: string, offset: number) {
const { view } = editorView()
act(() => {
view.dispatch(view.state.tr.insertText(text, offset + 1))
})
}
function focusComposerAt(start: number, end = start) {
const { view } = editorView()
vi.spyOn(view, 'hasFocus').mockReturnValue(true)
act(() => handle!.setSelectionOffsets(start, end))
}
const draft = () => screen.getByTestId('draft').textContent
const startButton = () => screen.getByRole('button', { name: en('voice.composer.start') })
const stopButton = () => screen.getByRole('button', { name: en('voice.composer.stop') })
async function beginRecording() {
await act(async () => {
fireEvent.click(startButton())
})
await screen.findByRole('button', { name: en('voice.composer.stop') })
}
/** Ends the recording and leaves transcription in flight. */
async function stopRecording() {
await act(async () => {
fireEvent.click(stopButton())
})
await screen.findByRole('button', { name: en('voice.composer.transcribing') })
}
async function deliver(text: string) {
await act(async () => {
finishTranscription(text)
})
}
beforeEach(() => {
vi.clearAllMocks()
handle = null
transcribeSignal = undefined
recording = fakeRecording()
mocks.supported.mockReturnValue(true)
mocks.startRecording.mockImplementation(async (options: StartRecordingOptions) => {
startOptions = options
return recording
})
mocks.transcribe.mockImplementation((_wav: Blob, options: { signal?: AbortSignal }) => {
transcribeSignal = options.signal
return new Promise((resolve, reject) => {
finishTranscription = (text) => resolve({ text, audioSeconds: 2, inferenceSeconds: 0.1 })
failTranscription = reject
})
})
useSettingsStore.setState({ locale: 'en' })
useVoiceInputStore.setState({ catalog: catalogFixture(), loading: false, error: null })
localStorage.clear()
// jsdom has no layout. ProseMirror reads Range geometry when a transaction
// scrolls the new selection into view.
Object.defineProperties(Range.prototype, {
getClientRects: { configurable: true, value: () => [] },
getBoundingClientRect: { configurable: true, value: () => new DOMRect() },
})
})
afterEach(() => {
Reflect.deleteProperty(Range.prototype, 'getClientRects')
Reflect.deleteProperty(Range.prototype, 'getBoundingClientRect')
cleanup()
vi.useRealTimers()
vi.restoreAllMocks()
})
describe('visibility', () => {
it('renders the dictation button when the service is ready', () => {
render(<Harness />)
expect(startButton()).toBeInTheDocument()
})
it.each([
['dictation is disabled', () => useVoiceInputStore.setState({ catalog: catalogFixture({ enabled: false }) })],
['the model is not downloaded', () => useVoiceInputStore.setState({ catalog: catalogFixture({}, 'unprepared') })],
['the platform does not support it', () => useVoiceInputStore.setState({ catalog: { ...catalogFixture(), supported: false } })],
['the environment cannot capture audio', () => mocks.supported.mockReturnValue(false)],
])('renders nothing and takes no space when %s', (_name, arrange) => {
arrange()
const { container } = render(<Harness />)
expect(screen.queryByTestId('voice-input')).toBeNull()
expect(container.querySelector('[data-testid="voice-input"]')).toBeNull()
})
it('loads the catalog when it mounts', async () => {
useVoiceInputStore.setState({ catalog: null })
mocks.catalog.mockResolvedValue(catalogFixture())
render(<Harness />)
expect(mocks.catalog).toHaveBeenCalledTimes(1)
expect(await screen.findByRole('button', { name: en('voice.composer.start') })).toBeInTheDocument()
})
it('keeps the button mounted mid-recording if the service is switched off meanwhile', async () => {
render(<Harness />)
await beginRecording()
act(() => useVoiceInputStore.setState({ catalog: catalogFixture({ enabled: false }) }))
expect(stopButton()).toBeInTheDocument()
})
it('does not steal focus from the composer on mouse down', () => {
render(<Harness />)
// fireEvent returns false when the default action was prevented.
expect(fireEvent.mouseDown(startButton())).toBe(false)
})
})
describe('recording and transcription', () => {
it('records with the remembered microphone and the configured limit, then transcribes with the selected provider and language', async () => {
localStorage.setItem('cc-haha-voice-input-device', 'usb-1')
useVoiceInputStore.setState({ catalog: catalogFixture({ language: 'en' }) })
render(<Harness />)
await beginRecording()
expect(startOptions).toMatchObject({ deviceId: 'usb-1', maxSeconds: 60 })
expect(screen.getByTestId('voice-input-timer')).toBeInTheDocument()
await stopRecording()
expect(mocks.transcribe).toHaveBeenCalledWith(expect.any(Blob), expect.objectContaining({
providerId: 'sensevoice-local',
language: 'en',
}))
await deliver('hello')
expect(draft()).toBe('hello')
expect(startButton()).toBeInTheDocument()
})
it('cannot start a second recording while one is running or being transcribed', async () => {
render(<Harness />)
await beginRecording()
await stopRecording()
expect(screen.getByRole('button', { name: en('voice.composer.transcribing') })).toBeDisabled()
expect(mocks.startRecording).toHaveBeenCalledTimes(1)
})
it('does not cancel a recording when the window loses focus', async () => {
render(<Harness />)
await beginRecording()
act(() => {
window.dispatchEvent(new Event('blur'))
document.dispatchEvent(new Event('visibilitychange'))
})
expect(recording.cancel).not.toHaveBeenCalled()
expect(stopButton()).toBeInTheDocument()
})
it('stops and transcribes on its own when the recorder reaches its limit', async () => {
render(<Harness />)
await beginRecording()
await act(async () => {
startOptions.onLimitReached?.()
})
expect(recording.stop).toHaveBeenCalledTimes(1)
await screen.findByRole('button', { name: en('voice.composer.transcribing') })
await deliver('long dictation')
expect(draft()).toBe('long dictation')
})
it('keeps recordings that are too short away from the server', async () => {
recording = fakeRecording(0.1)
render(<Harness />)
await beginRecording()
await act(async () => {
fireEvent.click(stopButton())
})
expect(mocks.transcribe).not.toHaveBeenCalled()
expect(await screen.findByRole('alert')).toHaveTextContent(en('voice.composer.error.tooShort'))
expect(draft()).toBe('')
})
it.each(['', ' ', '\n'])('reports silence instead of writing %j', async (text) => {
render(<Harness initial="keep" />)
await beginRecording()
await stopRecording()
await deliver(text)
expect(screen.getByRole('alert')).toHaveTextContent(en('voice.composer.error.noSpeech'))
expect(draft()).toBe('keep')
expect(screen.queryByTestId('voice-input-pending')).toBeNull()
})
})
describe('button semantics', () => {
it('is a pressed toggle only while recording; starting and transcribing are busy, not pressed', async () => {
let grant!: (value: FakeRecording) => void
mocks.startRecording.mockImplementation((options: StartRecordingOptions) => {
startOptions = options
return new Promise(resolve => { grant = resolve as typeof grant })
})
render(<Harness />)
expect(startButton()).not.toHaveAttribute('aria-pressed')
await act(async () => {
fireEvent.click(startButton())
})
const starting = screen.getByRole('button', { name: en('voice.composer.starting') })
expect(starting).toHaveAttribute('aria-busy', 'true')
expect(starting).not.toHaveAttribute('aria-pressed')
await act(async () => {
grant(recording)
})
expect(stopButton()).toHaveAttribute('aria-pressed', 'true')
await stopRecording()
const transcribing = screen.getByRole('button', { name: en('voice.composer.transcribing') })
expect(transcribing).toHaveAttribute('aria-busy', 'true')
expect(transcribing).not.toHaveAttribute('aria-pressed')
})
})
describe('cancellation', () => {
it('Esc discards the recording without transcribing', async () => {
render(<Harness />)
await beginRecording()
act(() => {
fireEvent.keyDown(window, { key: 'Escape' })
})
expect(recording.cancel).toHaveBeenCalledTimes(1)
expect(mocks.transcribe).not.toHaveBeenCalled()
expect(startButton()).toBeInTheDocument()
})
it('Esc during recognition aborts the request and ignores a late answer', async () => {
render(<Harness />)
await beginRecording()
await stopRecording()
act(() => {
fireEvent.keyDown(window, { key: 'Escape' })
})
expect(transcribeSignal?.aborted).toBe(true)
await deliver('too late')
expect(draft()).toBe('')
expect(screen.queryByTestId('voice-input-pending')).toBeNull()
})
it('Esc is left alone when nothing is being recorded', async () => {
render(<Harness />)
const event = new KeyboardEvent('keydown', { key: 'Escape', bubbles: true, cancelable: true })
window.dispatchEvent(event)
expect(event.defaultPrevented).toBe(false)
})
it('cancels a live recording when the composer unmounts', async () => {
const { unmount } = render(<Harness />)
await beginRecording()
unmount()
expect(recording.cancel).toHaveBeenCalledTimes(1)
})
it('cancels the microphone that arrives after the composer unmounted', async () => {
let grant!: (value: FakeRecording) => void
mocks.startRecording.mockImplementation((options: StartRecordingOptions) => {
startOptions = options
return new Promise(resolve => { grant = resolve as typeof grant })
})
const { unmount } = render(<Harness />)
await act(async () => {
fireEvent.click(startButton())
})
unmount()
expect(startOptions.signal?.aborted).toBe(true)
await act(async () => {
grant(recording)
})
expect(recording.cancel).toHaveBeenCalledTimes(1)
})
it('a second click while the permission prompt is open cancels the attempt', async () => {
let grant!: (value: FakeRecording) => void
mocks.startRecording.mockImplementation((options: StartRecordingOptions) => {
startOptions = options
return new Promise(resolve => { grant = resolve as typeof grant })
})
render(<Harness />)
await act(async () => {
fireEvent.click(startButton())
})
await act(async () => {
fireEvent.click(screen.getByRole('button', { name: en('voice.composer.starting') }))
})
await act(async () => {
grant(recording)
})
expect(recording.cancel).toHaveBeenCalledTimes(1)
expect(startButton()).toBeInTheDocument()
})
it('abandons the recording and any held text when the session changes', async () => {
const { rerender } = render(<Harness initial="abc" />)
await beginRecording()
rerender(<Harness initial="abc" contextKey="session-b" />)
expect(recording.cancel).toHaveBeenCalledTimes(1)
expect(startButton()).toBeInTheDocument()
})
it('drops a held result when the session changes', async () => {
const { rerender } = render(<Harness />)
await beginRecording()
await stopRecording()
typeText('x', 0)
await deliver('held')
expect(screen.getByTestId('voice-input-pending')).toBeInTheDocument()
rerender(<Harness contextKey="session-b" />)
expect(screen.queryByTestId('voice-input-pending')).toBeNull()
})
it('abandons dictation when the composer is hidden (contextKey null)', async () => {
const { rerender } = render(<Harness />)
await beginRecording()
rerender(<Harness contextKey={null} />)
expect(recording.cancel).toHaveBeenCalledTimes(1)
})
})
describe('write-back position', () => {
it('appends to the draft when the composer had no focus', async () => {
render(<Harness initial="你好" />)
await beginRecording()
await stopRecording()
await deliver('世界')
expect(draft()).toBe('你好世界')
})
it('writes at the caret captured when recording began, not where it is now', async () => {
render(<Harness initial="hello world" />)
focusComposerAt(5)
await beginRecording()
await stopRecording()
// The caret moves while the model is thinking; the text still belongs to
// the spot the user was speaking into.
act(() => handle!.setSelectionOffsets(11))
await deliver('big')
expect(draft()).toBe('hello big world')
})
it('replaces a selection that was active when recording began', async () => {
render(<Harness initial="fix the FOO now" />)
focusComposerAt(8, 11)
await beginRecording()
await stopRecording()
await deliver('bar')
expect(draft()).toBe('fix the bar now')
})
it('leaves the caret after the inserted text', async () => {
render(<Harness initial="ab" />)
focusComposerAt(1)
await beginRecording()
await stopRecording()
await deliver('你')
expect(handle!.getSelectionOffsets()).toEqual({ start: 2, end: 2 })
})
it('is one undoable editor step', async () => {
render(<Harness initial="keep" />)
await beginRecording()
await stopRecording()
await deliver('added')
expect(draft()).toBe('keep added')
const { view } = editorView()
act(() => {
undo(view.state, view.dispatch)
})
expect(draft()).toBe('keep')
})
it('never submits: writing text leaves the draft for the user to send', async () => {
const onSubmit = vi.fn()
render(<div onKeyDown={onSubmit}><Harness /></div>)
await beginRecording()
await stopRecording()
await deliver('hello')
expect(onSubmit).not.toHaveBeenCalled()
expect(draft()).toBe('hello')
})
})
describe('draft changed during recognition', () => {
it('keeps the text aside instead of overwriting the edit', async () => {
render(<Harness initial="hello" />)
await beginRecording()
await stopRecording()
typeText(' typed', 5)
await deliver('dictated')
expect(draft()).toBe('hello typed')
expect(screen.getByTestId('voice-input-pending-text')).toHaveTextContent('dictated')
})
it('treats an edit that was typed and deleted again as a change', async () => {
render(<Harness initial="hello" />)
await beginRecording()
typeText('x', 5)
const { view } = editorView()
act(() => {
view.dispatch(view.state.tr.delete(6, 7))
})
expect(draft()).toBe('hello')
await stopRecording()
await deliver('dictated')
expect(draft()).toBe('hello')
expect(screen.getByTestId('voice-input-pending')).toBeInTheDocument()
})
it('offers an insert button that writes at the caret the user has now', async () => {
render(<Harness initial="hello" />)
await beginRecording()
await stopRecording()
typeText(' typed', 5)
await deliver('dictated')
focusComposerAt(0)
fireEvent.click(screen.getByRole('button', { name: en('voice.composer.insertText') }))
// "dictated" is followed by "hello" without a space, so latin words are
// kept apart.
expect(draft()).toBe('dictated hello typed')
expect(screen.queryByTestId('voice-input-pending')).toBeNull()
})
it('lets the user discard the held text', async () => {
render(<Harness initial="hello" />)
await beginRecording()
await stopRecording()
typeText('!', 5)
await deliver('dictated')
fireEvent.click(screen.getByRole('button', { name: en('voice.composer.discard') }))
expect(screen.queryByTestId('voice-input-pending')).toBeNull()
expect(draft()).toBe('hello!')
})
it('discards stale held text when a new recording starts', async () => {
render(<Harness initial="hello" />)
await beginRecording()
await stopRecording()
typeText('!', 5)
await deliver('first')
expect(screen.getByTestId('voice-input-pending')).toBeInTheDocument()
await beginRecording()
expect(screen.queryByTestId('voice-input-pending')).toBeNull()
})
})
describe('composer blocked while a message is being sent', () => {
it('holds the result and only enables insertion once the composer frees up', async () => {
const { rerender } = render(<Harness initial="hello" />)
await beginRecording()
await stopRecording()
rerender(<Harness initial="hello" blocked />)
await deliver('dictated')
expect(draft()).toBe('hello')
const insert = screen.getByRole('button', { name: en('voice.composer.insertText') })
expect(insert).toBeDisabled()
rerender(<Harness initial="hello" />)
expect(screen.getByRole('button', { name: en('voice.composer.insertText') })).toBeEnabled()
fireEvent.click(screen.getByRole('button', { name: en('voice.composer.insertText') }))
expect(draft()).toBe('hello dictated')
})
})
describe('IME composition', () => {
it('holds a result that arrives mid-composition and never writes it on its own', async () => {
vi.useFakeTimers({ shouldAdvanceTime: true })
render(<Harness initial="hello" />)
await beginRecording()
await stopRecording()
const { editor } = editorView()
fireEvent.compositionStart(editor)
await deliver('你好')
expect(draft()).toBe('hello')
expect(screen.getByTestId('voice-input-pending-text')).toHaveTextContent('你好')
// Ending the composition does not release it: the user chooses.
fireEvent.compositionEnd(editor)
await act(async () => {
await vi.advanceTimersByTimeAsync(200)
})
expect(draft()).toBe('hello')
expect(screen.getByTestId('voice-input-pending')).toBeInTheDocument()
fireEvent.click(screen.getByRole('button', { name: en('voice.composer.insertText') }))
expect(draft()).toBe('hello你好')
})
})
describe('errors', () => {
it.each<[string, VoiceRecorderError['code'], TranslationKey]>([
['permission', 'permission', 'voice.composer.error.permission'],
['no device', 'no-device', 'voice.composer.error.noDevice'],
['device busy', 'device-busy', 'voice.composer.error.deviceBusy'],
['unavailable', 'unavailable', 'voice.composer.error.unavailable'],
['failed', 'failed', 'voice.composer.error.failed'],
])('shows a specific message when the microphone fails to open (%s)', async (_label, code, key) => {
mocks.startRecording.mockRejectedValue(new VoiceRecorderError(code))
render(<Harness />)
await act(async () => {
fireEvent.click(startButton())
})
expect(await screen.findByRole('alert')).toHaveTextContent(en(key))
expect(startButton()).toBeInTheDocument()
})
it('shows a message when the microphone drops out mid-recording', async () => {
render(<Harness />)
await beginRecording()
act(() => {
startOptions.onInterrupted?.(new VoiceRecorderError('interrupted'))
})
expect(screen.getByRole('alert')).toHaveTextContent(en('voice.composer.error.interrupted'))
expect(startButton()).toBeInTheDocument()
})
it.each<[string, unknown, TranslationKey]>([
['not-ready body', new ApiError(409, { error: 'voice/not-ready', message: 'x' }), 'voice.composer.error.notReady'],
['invalid-audio body', new ApiError(400, { error: 'voice/invalid-audio', message: 'x' }), 'voice.composer.error.invalidAudio'],
['unknown-provider body', new ApiError(404, { error: 'voice/unknown-provider', message: 'x' }), 'voice.composer.error.unknownProvider'],
['failed body', new ApiError(500, { error: 'voice/failed', message: 'x' }), 'voice.composer.error.failed'],
['bare 409', new ApiError(409, 'conflict'), 'voice.composer.error.notReady'],
['network error', new TypeError('Failed to fetch'), 'voice.composer.error.failed'],
])('maps a server response to a message (%s)', async (_label, error, key) => {
render(<Harness />)
await beginRecording()
await stopRecording()
await act(async () => {
failTranscription(error)
})
expect(screen.getByRole('alert')).toHaveTextContent(en(key))
expect(startButton()).toBeInTheDocument()
})
it('refreshes the catalog when the server says the model is not ready, so a deleted model stops offering the button', async () => {
mocks.catalog.mockResolvedValue(catalogFixture({}, 'unprepared'))
render(<Harness />)
await beginRecording()
await stopRecording()
await act(async () => {
failTranscription(new ApiError(409, { error: 'voice/not-ready', message: 'x' }))
})
expect(mocks.catalog).toHaveBeenCalledTimes(1)
await waitFor(() => expect(selectVoiceInputReady(useVoiceInputStore.getState())).toBe(false))
})
it('does not refetch the catalog for other failures', async () => {
render(<Harness />)
await beginRecording()
await stopRecording()
await act(async () => {
failTranscription(new ApiError(500, { error: 'voice/failed', message: 'x' }))
})
expect(mocks.catalog).not.toHaveBeenCalled()
})
it('can be dismissed, and clears itself after a while', async () => {
vi.useFakeTimers({ shouldAdvanceTime: true })
mocks.startRecording.mockRejectedValue(new VoiceRecorderError('no-device'))
render(<Harness />)
await act(async () => {
fireEvent.click(startButton())
})
fireEvent.click(screen.getByRole('button', { name: en('voice.composer.dismiss') }))
expect(screen.queryByRole('alert')).toBeNull()
await act(async () => {
fireEvent.click(startButton())
})
expect(screen.getByRole('alert')).toBeInTheDocument()
await act(async () => {
await vi.advanceTimersByTimeAsync(9000)
})
await waitFor(() => expect(screen.queryByRole('alert')).toBeNull())
})
})
@@ -0,0 +1,191 @@
import { useEffect, useRef, useState } from 'react'
import { Button } from '@/components/ui/Button'
import { IconButton } from '@/components/ui/IconButton'
import { useTranslation } from '@/i18n'
import type { TranslationKey } from '@/i18n/locales/en'
import { selectVoiceInputReady, useVoiceInputStore } from '@/stores/voiceInputStore'
import { isVoiceCaptureSupported } from './recorder'
import type { ComposerDictation, DictationIssue } from './useComposerDictation'
type VoiceInputButtonProps = {
dictation: ComposerDictation
/** The composer cannot take text right now; a held result must wait. */
blocked?: boolean
/** 44px touch target, matching the composer's other mobile controls. */
mobile?: boolean
}
const ISSUE_KEYS: Record<DictationIssue, TranslationKey> = {
permission: 'voice.composer.error.permission',
noDevice: 'voice.composer.error.noDevice',
deviceBusy: 'voice.composer.error.deviceBusy',
unavailable: 'voice.composer.error.unavailable',
interrupted: 'voice.composer.error.interrupted',
notReady: 'voice.composer.error.notReady',
invalidAudio: 'voice.composer.error.invalidAudio',
unknownProvider: 'voice.composer.error.unknownProvider',
failed: 'voice.composer.error.failed',
noSpeech: 'voice.composer.error.noSpeech',
tooShort: 'voice.composer.error.tooShort',
}
/** Nothing was wrong; there was just nothing to write. */
const SOFT_ISSUES = new Set<DictationIssue>(['noSpeech', 'tooShort'])
function formatElapsed(ms: number): string {
const total = Math.max(0, Math.floor(ms / 1000))
return `${Math.floor(total / 60)}:${String(total % 60).padStart(2, '0')}`
}
/**
* The composer's dictation control. Renders nothing until the voice service is
* enabled, its model is downloaded, and this environment can capture audio.
*/
export function VoiceInputButton({ dictation, blocked = false, mobile = false }: VoiceInputButtonProps) {
const t = useTranslation()
const ready = useVoiceInputStore(selectVoiceInputReady)
const loadCatalog = useVoiceInputStore(state => state.loadCatalog)
const [supported] = useState(isVoiceCaptureSupported)
const { phase, issue, pendingText, startedAt, getLevel } = dictation
const haloRef = useRef<HTMLSpanElement>(null)
const [elapsed, setElapsed] = useState(0)
useEffect(() => {
void loadCatalog()
}, [loadCatalog])
// Loudness drives the halo straight through the DOM: a 60 Hz value has no
// business re-rendering the composer.
useEffect(() => {
if (phase !== 'recording') return
let frame = 0
const tick = () => {
const halo = haloRef.current
if (halo) {
const level = getLevel()
halo.style.transform = `scale(${1 + level * 0.6})`
halo.style.opacity = String(0.25 + level * 0.75)
}
frame = requestAnimationFrame(tick)
}
frame = requestAnimationFrame(tick)
return () => cancelAnimationFrame(frame)
}, [getLevel, phase])
useEffect(() => {
if (phase !== 'recording') return
setElapsed(0)
const timer = setInterval(() => setElapsed(Date.now() - startedAt), 250)
return () => clearInterval(timer)
}, [phase, startedAt])
const engaged = phase !== 'idle' || pendingText !== null || issue !== null
if (!supported || (!ready && !engaged)) return null
const size = mobile ? '2xl' : 'md'
const recording = phase === 'recording'
const label = phase === 'idle'
? t('voice.composer.start')
: phase === 'starting'
? t('voice.composer.starting')
: recording
? t('voice.composer.stop')
: t('voice.composer.transcribing')
return (
<div data-testid="voice-input" className="relative flex shrink-0 items-center gap-1.5">
{recording && (
<span
data-testid="voice-input-timer"
className="text-xs tabular-nums text-[var(--color-error)]"
title={t('voice.composer.recordingHint')}
>
{formatElapsed(elapsed)}
</span>
)}
<span className="relative inline-flex">
{recording && (
<span
ref={haloRef}
aria-hidden="true"
className="pointer-events-none absolute inset-0 rounded-[var(--radius-lg)] bg-[var(--color-error-soft)]"
/>
)}
<IconButton
icon={recording ? 'stop' : 'mic'}
label={label}
size={size}
tone={recording ? 'danger' : 'secondary'}
solid={recording}
loading={phase === 'transcribing'}
// A toggle only while it is recording. The label changes with the
// phase, so pressed would contradict it during the other phases;
// those are busy instead (`loading` already sets it for transcribing).
pressed={recording ? true : undefined}
aria-busy={phase === 'starting' || phase === 'transcribing' ? true : undefined}
// Keep the caret in the composer: a click would otherwise blur it, and
// the write-back position is the caret the user left there.
onMouseDown={event => event.preventDefault()}
onClick={dictation.toggle}
className="relative"
/>
</span>
{issue && (
<div
role="alert"
data-testid="voice-input-issue"
className={[
'absolute bottom-full right-0 z-[var(--z-popover)] mb-2 flex w-max max-w-[min(20rem,calc(100vw-2rem))] items-start gap-2',
'rounded-[var(--radius-lg)] px-3 py-2 text-xs shadow-[var(--shadow-overlay)]',
SOFT_ISSUES.has(issue)
? 'bg-[var(--color-warning-container)] text-[var(--color-on-warning-container)]'
: 'bg-[var(--color-error-container)] text-[var(--color-on-error-container)]',
].join(' ')}
>
<span className="min-w-0 flex-1">{t(ISSUE_KEYS[issue])}</span>
<button
type="button"
aria-label={t('voice.composer.dismiss')}
onClick={dictation.dismissIssue}
className="shrink-0 rounded-[var(--radius-sm)] focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[var(--color-border-focus)]"
>
<span className="material-symbols-outlined text-[14px]" aria-hidden="true">close</span>
</button>
</div>
)}
{pendingText !== null && (
<div
role="group"
aria-label={t('voice.composer.pendingTitle')}
data-testid="voice-input-pending"
className={[
'absolute bottom-full right-0 z-[var(--z-popover)] mb-2 flex w-72 max-w-[calc(100vw-2rem)] flex-col gap-2',
'rounded-[var(--radius-lg)] border border-[var(--color-border)] bg-[var(--color-surface-container-lowest)] p-3',
'shadow-[var(--shadow-overlay)]',
].join(' ')}
>
<p className="text-xs text-[var(--color-text-tertiary)]">{t('voice.composer.pendingHint')}</p>
<p data-testid="voice-input-pending-text" className="line-clamp-4 break-words text-sm text-[var(--color-text-primary)]">
{pendingText}
</p>
<div className="flex justify-end gap-2">
<Button variant="ghost" size="sm" onClick={dictation.dismissPending}>
{t('voice.composer.discard')}
</Button>
<Button
variant="tonal"
size="sm"
disabled={blocked}
onMouseDown={event => event.preventDefault()}
onClick={dictation.insertPending}
>
{t('voice.composer.insertText')}
</Button>
</div>
</div>
)}
</div>
)
}
@@ -0,0 +1,302 @@
import { act, cleanup, render } from '@testing-library/react'
import '@testing-library/jest-dom'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { VoiceWave } from './VoiceWave'
const HEIGHT = 44
const MIDDLE = HEIGHT / 2
type FakeContext = {
setTransform: ReturnType<typeof vi.fn>
clearRect: ReturnType<typeof vi.fn>
beginPath: ReturnType<typeof vi.fn>
moveTo: ReturnType<typeof vi.fn>
lineTo: ReturnType<typeof vi.fn>
stroke: ReturnType<typeof vi.fn>
globalAlpha: number
lineWidth: number
strokeStyle: string
lineCap: string
lineJoin: string
}
let context: FakeContext
let frames: Map<number, FrameRequestCallback>
let nextFrameId: number
let cancelFrame: ReturnType<typeof vi.fn>
let disconnect: ReturnType<typeof vi.fn>
let resize: (width: number) => void
let reducedMotion: boolean
let hidden: boolean
function makeContext(): FakeContext {
return {
setTransform: vi.fn(), clearRect: vi.fn(), beginPath: vi.fn(), moveTo: vi.fn(), lineTo: vi.fn(), stroke: vi.fn(),
globalAlpha: 1, lineWidth: 1, strokeStyle: '', lineCap: '', lineJoin: '',
}
}
/** Runs the queued frames once, `now` milliseconds on the animation clock. */
function runFrames(now: number) {
const batch = [...frames.values()]
frames.clear()
act(() => { for (const callback of batch) callback(now) })
}
/** Largest distance any drawn point of the last stroked layer reached from the centre line. */
function peakOffset(): number {
return Math.max(...context.lineTo.mock.calls.map(([, y]) => Math.abs((y as number) - MIDDLE)))
}
/** Runs one frame and returns the y values the first (top) layer drew in it. */
function topLayerAt(now: number): number[] {
context.moveTo.mockClear()
context.lineTo.mockClear()
runFrames(now)
const points = context.lineTo.mock.calls
return points.slice(0, points.length / 3).map(([, y]) => y as number)
}
function expectClose(actual: number[], expected: number[]) {
expect(actual).toHaveLength(expected.length)
actual.forEach((y, index) => expect(y).toBeCloseTo(expected[index]!, 6))
}
beforeEach(() => {
context = makeContext()
frames = new Map()
nextFrameId = 1
cancelFrame = vi.fn((id: number) => { frames.delete(id) })
disconnect = vi.fn()
reducedMotion = false
hidden = false
vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockImplementation(() => context as never)
Object.defineProperty(HTMLCanvasElement.prototype, 'clientWidth', { configurable: true, get: () => 300 })
vi.stubGlobal('requestAnimationFrame', (callback: FrameRequestCallback) => {
const id = nextFrameId++
frames.set(id, callback)
return id
})
vi.stubGlobal('cancelAnimationFrame', cancelFrame)
vi.stubGlobal('ResizeObserver', class {
constructor(callback: ResizeObserverCallback) {
resize = (width) => callback([{ contentRect: { width } } as ResizeObserverEntry], this as unknown as ResizeObserver)
}
observe() {}
unobserve() {}
disconnect = disconnect
})
vi.stubGlobal('matchMedia', (query: string) => ({
get matches() { return query.includes('prefers-reduced-motion') && reducedMotion },
}))
Object.defineProperty(document, 'hidden', { configurable: true, get: () => hidden })
vi.stubGlobal('devicePixelRatio', 2)
})
afterEach(() => {
cleanup()
Reflect.deleteProperty(HTMLCanvasElement.prototype, 'clientWidth')
Reflect.deleteProperty(document, 'hidden')
vi.unstubAllGlobals()
vi.restoreAllMocks()
})
describe('VoiceWave', () => {
it('reads the level and draws three layers on every frame while active', () => {
const getLevel = vi.fn(() => 0.5)
const layers: Array<{ alpha: number; width: number }> = []
context.stroke.mockImplementation(() => { layers.push({ alpha: context.globalAlpha, width: context.lineWidth }) })
render(<VoiceWave getLevel={getLevel} active />)
layers.length = 0
expect(frames.size).toBe(1)
runFrames(1_000)
expect(getLevel).toHaveBeenCalledTimes(1)
// The top layer is opaque and full width; the lower two are fainter and thinner.
expect(layers).toEqual([
{ alpha: 1, width: 2 },
{ alpha: 0.42, width: 1.25 },
{ alpha: 0.26, width: 1 },
])
expect(context.lineCap).toBe('round')
runFrames(1_016)
expect(getLevel).toHaveBeenCalledTimes(2)
expect(layers).toHaveLength(6)
})
it('marks the canvas decorative and sizes it to the container at the device pixel ratio', () => {
const { container } = render(<VoiceWave getLevel={() => 0} active />)
const canvas = container.querySelector('canvas')!
expect(canvas).toHaveAttribute('aria-hidden', 'true')
expect(canvas.style.height).toBe('44px')
// 300 css px wide, 44 tall, at 2x.
expect(canvas.width).toBe(600)
expect(canvas.height).toBe(88)
expect(context.setTransform).toHaveBeenLastCalledWith(2, 0, 0, 2, 0, 0)
act(() => resize(420))
expect(canvas.width).toBe(840)
expect(canvas.height).toBe(88)
})
it('pins both ends to the centre line so the wave floats in the middle', () => {
render(<VoiceWave getLevel={() => 1} active />)
runFrames(1_000)
runFrames(1_500)
const first = context.moveTo.mock.calls[0]!
expect(first[0]).toBe(0)
expect(first[1]).toBeCloseTo(MIDDLE, 5)
const last = context.lineTo.mock.calls.filter(([x]) => x === 300)[0]!
expect(last[1]).toBeCloseTo(MIDDLE, 5)
// ... while the middle of the wave does swing.
expect(peakOffset()).toBeGreaterThan(8)
})
it('rises quickly on loud input and falls back slowly', () => {
let level = 0
render(<VoiceWave getLevel={() => level} active />)
runFrames(1_000)
context.lineTo.mockClear()
runFrames(1_016)
const idle = peakOffset()
level = 1
context.lineTo.mockClear()
runFrames(1_032)
const attacked = peakOffset()
// One 16 ms frame already covers a good part of the way up.
expect(attacked).toBeGreaterThan(idle + 3)
for (let time = 1_048; time < 1_300; time += 16) runFrames(time)
context.lineTo.mockClear()
runFrames(1_316)
const loud = peakOffset()
level = 0
context.lineTo.mockClear()
runFrames(1_332)
const released = peakOffset()
// Still most of the way up one frame after the input went quiet ...
expect(released).toBeGreaterThan(loud * 0.8)
// ... and settled back down a couple of seconds later.
for (let time = 1_348; time < 3_400; time += 16) runFrames(time)
context.lineTo.mockClear()
runFrames(3_416)
expect(peakOffset()).toBeLessThan(idle + 1)
})
it('keeps a near-flat line when there is no sound', () => {
render(<VoiceWave getLevel={() => 0} active />)
runFrames(1_000)
runFrames(1_016)
expect(peakOffset()).toBeGreaterThan(0.5)
expect(peakOffset()).toBeLessThan(3)
})
it('draws one calm frame and schedules nothing while inactive', () => {
const getLevel = vi.fn(() => 1)
render(<VoiceWave getLevel={getLevel} active={false} />)
expect(frames.size).toBe(0)
expect(getLevel).not.toHaveBeenCalled()
// A single static paint at mount: three layers, nothing more.
expect(context.stroke).toHaveBeenCalledTimes(3)
expect(peakOffset()).toBeLessThan(3)
})
it('stops reading the level and cancels the frame when it goes inactive', () => {
const getLevel = vi.fn(() => 0.5)
const { rerender } = render(<VoiceWave getLevel={getLevel} active />)
runFrames(1_000)
const readsWhileActive = getLevel.mock.calls.length
rerender(<VoiceWave getLevel={getLevel} active={false} />)
expect(cancelFrame).toHaveBeenCalled()
expect(frames.size).toBe(0)
runFrames(1_016)
expect(getLevel).toHaveBeenCalledTimes(readsWhileActive)
})
it('cancels the frame, disconnects the observer and stops reading on unmount', () => {
const getLevel = vi.fn(() => 0.5)
const { unmount } = render(<VoiceWave getLevel={getLevel} active />)
runFrames(1_000)
const reads = getLevel.mock.calls.length
unmount()
expect(cancelFrame).toHaveBeenCalled()
expect(disconnect).toHaveBeenCalled()
expect(frames.size).toBe(0)
expect(getLevel).toHaveBeenCalledTimes(reads)
})
it('does not restart the loop when the parent passes a new getLevel function', () => {
const first = vi.fn(() => 0.2)
const second = vi.fn(() => 0.8)
const { rerender } = render(<VoiceWave getLevel={first} active />)
runFrames(1_000)
rerender(<VoiceWave getLevel={second} active />)
expect(cancelFrame).not.toHaveBeenCalled()
runFrames(1_016)
expect(second).toHaveBeenCalled()
expect(first).toHaveBeenCalledTimes(1)
})
it('does not throw or animate when the canvas has no 2D context', () => {
vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(null)
expect(() => render(<VoiceWave getLevel={() => 1} active />)).not.toThrow()
expect(frames.size).toBe(0)
})
it('does not spin while the window is hidden, and resumes when it returns', () => {
hidden = true
const getLevel = vi.fn(() => 0.5)
render(<VoiceWave getLevel={getLevel} active />)
expect(frames.size).toBe(0)
hidden = false
act(() => { document.dispatchEvent(new Event('visibilitychange')) })
expect(frames.size).toBe(1)
runFrames(1_000)
expect(getLevel).toHaveBeenCalledTimes(1)
hidden = true
act(() => { document.dispatchEvent(new Event('visibilitychange')) })
expect(frames.size).toBe(0)
expect(cancelFrame).toHaveBeenCalled()
})
it('flows forward over time normally', () => {
render(<VoiceWave getLevel={() => 0.6} active />)
runFrames(1_000)
const before = topLayerAt(1_100)
const after = topLayerAt(1_200)
expect(after).not.toEqual(before)
})
it('holds the phase under prefers-reduced-motion and only lets the level change the height', () => {
reducedMotion = true
let level = 0.6
render(<VoiceWave getLevel={() => level} active />)
// Let the smoothed level settle so only the phase could differ between frames.
for (let time = 1_000; time < 3_000; time += 16) runFrames(time)
const before = topLayerAt(3_016)
const after = topLayerAt(3_500)
expectClose(after, before)
level = 1
for (let time = 3_516; time < 5_000; time += 16) runFrames(time)
const louder = topLayerAt(5_016)
expect(Math.max(...louder.map(y => Math.abs(y - MIDDLE)))).toBeGreaterThan(Math.max(...after.map(y => Math.abs(y - MIDDLE))))
})
})
@@ -0,0 +1,153 @@
import { memo, useEffect, useRef } from 'react'
const HEIGHT = 44
/** Idle undulation: the line stays visibly alive, but reads as flat. */
const BASE_AMPLITUDE = 0.04 * HEIGHT
/** Level smoothing time constants: rise quickly on speech, fall back slowly. */
const ATTACK_SECONDS = 0.06
const RELEASE_SECONDS = 0.35
/** Longest step folded into one frame, so a stalled tab does not lurch on return. */
const MAX_FRAME_SECONDS = 0.1
const POINT_SPACING = 2
/**
* Three overlapping sine waves. Only the top one is fully opaque; the others
* are thinner and fainter so the shape reads as one motion, not three lines.
* `speed` is radians per second; all flow the same way at slightly different
* rates, which keeps the layers drifting against each other.
*/
const LAYERS = [
{ wavelength: 150, speed: 1.8, offset: 0, scale: 1, lineWidth: 2, alpha: 1 },
{ wavelength: 104, speed: 1.3, offset: 1.7, scale: 0.68, lineWidth: 1.25, alpha: 0.42 },
{ wavelength: 76, speed: 2.4, offset: 3.4, scale: 0.48, lineWidth: 1, alpha: 0.26 },
] as const
type Props = {
/** Input loudness in 0..1. Read every frame from a ref, so a new function identity never restarts the loop. */
getLevel: () => number
/** Animate from `getLevel`. Inactive draws a single calm line and does no work. */
active: boolean
className?: string
}
function readColor(canvas: HTMLCanvasElement): string {
const style = getComputedStyle(canvas)
return style.getPropertyValue('--color-brand').trim() || style.color
}
/**
* A quiet, layered waveform for the recording level.
*
* Drawn straight onto a canvas from requestAnimationFrame: level changes never
* touch React state, so the parent does not re-render 60 times a second.
*/
export const VoiceWave = memo(function VoiceWave({ getLevel, active, className }: Props) {
const canvasRef = useRef<HTMLCanvasElement>(null)
const getLevelRef = useRef(getLevel)
getLevelRef.current = getLevel
useEffect(() => {
const canvas = canvasRef.current
const context = canvas?.getContext('2d') ?? null
if (!canvas || !context) return
const reducedMotion = typeof window.matchMedia === 'function'
? window.matchMedia('(prefers-reduced-motion: reduce)')
: null
let width = 0
let pixelRatio = 1
let color = readColor(canvas)
let frame: number | null = null
let lastTime = 0
let phase = 0
let level = 0
const paint = () => {
if (width <= 0) return
context.setTransform(pixelRatio, 0, 0, pixelRatio, 0, 0)
context.clearRect(0, 0, width, HEIGHT)
context.lineCap = 'round'
context.lineJoin = 'round'
context.strokeStyle = color
const middle = HEIGHT / 2
const amplitude = BASE_AMPLITUDE + level * (middle - BASE_AMPLITUDE - 2)
for (const layer of LAYERS) {
context.globalAlpha = layer.alpha
context.lineWidth = layer.lineWidth
context.beginPath()
for (let x = 0; x <= width; x += POINT_SPACING) {
// The envelope pins both ends to the centre line, so the wave floats
// in the middle instead of hitting the edges.
const envelope = Math.pow(Math.sin((Math.PI * x) / width), 1.6)
const angle = (x / layer.wavelength) * Math.PI * 2 - phase * layer.speed + layer.offset
const y = middle + Math.sin(angle) * amplitude * layer.scale * envelope
if (x === 0) context.moveTo(x, y)
else context.lineTo(x, y)
}
context.stroke()
}
context.globalAlpha = 1
}
const layout = (nextWidth: number) => {
width = Math.max(0, Math.round(nextWidth))
pixelRatio = window.devicePixelRatio || 1
canvas.width = Math.round(width * pixelRatio)
canvas.height = Math.round(HEIGHT * pixelRatio)
color = readColor(canvas)
paint()
}
const tick = (now: number) => {
const elapsed = lastTime ? Math.min(MAX_FRAME_SECONDS, (now - lastTime) / 1000) : 0
lastTime = now
const target = Math.max(0, Math.min(1, getLevelRef.current()))
const seconds = target > level ? ATTACK_SECONDS : RELEASE_SECONDS
level += (target - level) * (1 - Math.exp(-elapsed / seconds))
if (!reducedMotion?.matches) phase += elapsed
paint()
frame = requestAnimationFrame(tick)
}
const start = () => {
if (frame !== null || document.hidden) return
lastTime = 0
frame = requestAnimationFrame(tick)
}
const stop = () => {
if (frame === null) return
cancelAnimationFrame(frame)
frame = null
}
// A hidden window would spin the loop for nothing; resume when it returns.
const onVisibilityChange = () => (document.hidden ? stop() : start())
const observer = typeof ResizeObserver === 'function'
? new ResizeObserver((entries) => {
const entry = entries[entries.length - 1]
if (entry) layout(entry.contentRect.width)
})
: null
observer?.observe(canvas)
layout(canvas.clientWidth)
if (active) {
document.addEventListener('visibilitychange', onVisibilityChange)
start()
}
return () => {
stop()
observer?.disconnect()
document.removeEventListener('visibilitychange', onVisibilityChange)
}
}, [active])
return (
<canvas
ref={canvasRef}
aria-hidden="true"
className={className ? `block w-full ${className}` : 'block w-full'}
style={{ height: HEIGHT }}
/>
)
})
@@ -0,0 +1,42 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { getPreferredMicrophoneId, setPreferredMicrophoneId } from './devicePreference'
const KEY = 'cc-haha-voice-input-device'
describe('microphone preference', () => {
beforeEach(() => {
localStorage.clear()
})
afterEach(() => {
vi.restoreAllMocks()
})
it('round-trips the chosen device id under the documented key', () => {
setPreferredMicrophoneId('mic-2')
expect(localStorage.getItem(KEY)).toBe('mic-2')
expect(getPreferredMicrophoneId()).toBe('mic-2')
})
it('clears the preference when set to undefined', () => {
setPreferredMicrophoneId('mic-2')
setPreferredMicrophoneId(undefined)
expect(localStorage.getItem(KEY)).toBeNull()
expect(getPreferredMicrophoneId()).toBeUndefined()
})
it('treats an empty stored value as no preference', () => {
localStorage.setItem(KEY, '')
expect(getPreferredMicrophoneId()).toBeUndefined()
})
it('does not throw when storage is unavailable', () => {
vi.spyOn(Storage.prototype, 'getItem').mockImplementation(() => { throw new Error('blocked') })
vi.spyOn(Storage.prototype, 'setItem').mockImplementation(() => { throw new Error('blocked') })
vi.spyOn(Storage.prototype, 'removeItem').mockImplementation(() => { throw new Error('blocked') })
expect(getPreferredMicrophoneId()).toBeUndefined()
expect(() => setPreferredMicrophoneId('mic-2')).not.toThrow()
expect(() => setPreferredMicrophoneId(undefined)).not.toThrow()
})
})
@@ -0,0 +1,19 @@
const STORAGE_KEY = 'cc-haha-voice-input-device'
/** The microphone the user picked in settings; undefined means the system default. */
export function getPreferredMicrophoneId(): string | undefined {
try {
return localStorage.getItem(STORAGE_KEY) || undefined
} catch {
return undefined
}
}
export function setPreferredMicrophoneId(id: string | undefined): void {
try {
if (id) localStorage.setItem(STORAGE_KEY, id)
else localStorage.removeItem(STORAGE_KEY)
} catch {
// Storage can be blocked or full; the choice then lasts only until reload.
}
}
@@ -0,0 +1,118 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { listAudioInputs } from './devices'
const originalMediaDevices = Object.getOwnPropertyDescriptor(navigator, 'mediaDevices')
function device(kind: string, deviceId: string, label: string) {
return { kind, deviceId, label } as MediaDeviceInfo
}
function stubMediaDevices(value: unknown) {
Object.defineProperty(navigator, 'mediaDevices', { value, configurable: true })
}
describe('listAudioInputs', () => {
beforeEach(() => {
stubMediaDevices(undefined)
})
afterEach(() => {
if (originalMediaDevices) Object.defineProperty(navigator, 'mediaDevices', originalMediaDevices)
else delete (navigator as { mediaDevices?: unknown }).mediaDevices
})
it('returns an empty list when the browser has no media devices API', async () => {
await expect(listAudioInputs()).resolves.toEqual([])
})
it('keeps only audio inputs and drops the communications alias', async () => {
stubMediaDevices({
enumerateDevices: vi.fn(async () => [
device('videoinput', 'cam', 'Camera'),
device('audiooutput', 'spk', 'Speakers'),
device('audioinput', 'default', 'Default - Built-in'),
device('audioinput', 'communications', 'Communications'),
device('audioinput', 'usb', 'USB Mic'),
]),
})
await expect(listAudioInputs()).resolves.toEqual([
{ deviceId: 'default', label: 'Default - Built-in' },
{ deviceId: 'usb', label: 'USB Mic' },
])
})
it('does not open a stream unless permission is requested', async () => {
const getUserMedia = vi.fn()
stubMediaDevices({
enumerateDevices: vi.fn(async () => [device('audioinput', 'a', '')]),
getUserMedia,
})
await expect(listAudioInputs()).resolves.toEqual([{ deviceId: 'a', label: '' }])
expect(getUserMedia).not.toHaveBeenCalled()
})
it('opens and immediately closes a stream to unlock labels', async () => {
const stop = vi.fn()
const getUserMedia = vi.fn(async () => ({ getTracks: () => [{ stop }, { stop }] }))
const enumerateDevices = vi.fn()
.mockResolvedValueOnce([device('audioinput', 'a', '')])
.mockResolvedValueOnce([device('audioinput', 'a', 'Studio Mic')])
stubMediaDevices({ enumerateDevices, getUserMedia })
await expect(listAudioInputs({ requestPermission: true })).resolves.toEqual([
{ deviceId: 'a', label: 'Studio Mic' },
])
expect(getUserMedia).toHaveBeenCalledWith({ audio: true })
expect(stop).toHaveBeenCalledTimes(2)
})
it('skips the permission prompt when labels are already readable', async () => {
const getUserMedia = vi.fn()
stubMediaDevices({
enumerateDevices: vi.fn(async () => [device('audioinput', 'a', 'Studio Mic')]),
getUserMedia,
})
await listAudioInputs({ requestPermission: true })
expect(getUserMedia).not.toHaveBeenCalled()
})
it('returns the unlabeled list when permission is denied', async () => {
stubMediaDevices({
enumerateDevices: vi.fn(async () => [device('audioinput', 'a', '')]),
getUserMedia: vi.fn(async () => { throw new DOMException('denied', 'NotAllowedError') }),
})
await expect(listAudioInputs({ requestPermission: true })).resolves.toEqual([{ deviceId: 'a', label: '' }])
})
it('reports why the permission stream failed without changing the returned list', async () => {
const denied = new DOMException('denied', 'NotAllowedError')
const onPermissionError = vi.fn()
stubMediaDevices({
enumerateDevices: vi.fn(async () => [device('audioinput', '', '')]),
getUserMedia: vi.fn(async () => { throw denied }),
})
await expect(listAudioInputs({ requestPermission: true, onPermissionError })).resolves.toEqual([{ deviceId: '', label: '' }])
expect(onPermissionError).toHaveBeenCalledWith(denied)
})
it('does not report a permission error when the stream opens', async () => {
const onPermissionError = vi.fn()
stubMediaDevices({
enumerateDevices: vi.fn(async () => [device('audioinput', 'a', '')]),
getUserMedia: vi.fn(async () => ({ getTracks: () => [] })),
})
await listAudioInputs({ requestPermission: true, onPermissionError })
expect(onPermissionError).not.toHaveBeenCalled()
})
it('returns an empty list when enumeration itself fails', async () => {
stubMediaDevices({ enumerateDevices: vi.fn(async () => { throw new Error('boom') }) })
await expect(listAudioInputs()).resolves.toEqual([])
})
})
@@ -0,0 +1,41 @@
export type AudioInputDevice = { deviceId: string; label: string }
function readInputs(devices: MediaDeviceInfo[]): AudioInputDevice[] {
return devices
.filter(device => device.kind === 'audioinput')
// Chromium adds a virtual "communications" alias of another entry.
.filter(device => device.deviceId !== 'communications')
.map(device => ({ deviceId: device.deviceId, label: device.label }))
}
/**
* Lists microphones. Labels stay empty until the page holds a microphone
* permission; `requestPermission` opens a stream just long enough to unlock
* them and closes it again before returning.
*/
export async function listAudioInputs(options?: {
requestPermission?: boolean
/**
* Called with the `getUserMedia` failure when the permission stream cannot
* open. The returned list is unchanged; this only tells the caller why the
* labels stayed hidden (denied, no device, device busy).
*/
onPermissionError?: (error: unknown) => void
}): Promise<AudioInputDevice[]> {
const mediaDevices = typeof navigator === 'undefined' ? undefined : navigator.mediaDevices
if (!mediaDevices?.enumerateDevices) return []
let inputs = readInputs(await mediaDevices.enumerateDevices().catch(() => []))
const needsLabels = inputs.length === 0 || inputs.some(device => !device.label)
if (options?.requestPermission && needsLabels && mediaDevices.getUserMedia) {
try {
const stream = await mediaDevices.getUserMedia({ audio: true })
for (const track of stream.getTracks()) track.stop()
inputs = readInputs(await mediaDevices.enumerateDevices().catch(() => []))
} catch (error) {
// Denied or nothing plugged in: report the unlabeled list we already have.
options.onPermissionError?.(error)
}
}
return inputs
}
@@ -0,0 +1,59 @@
import { describe, expect, it } from 'vitest'
import {
normalizeDictationText,
placeDictationResult,
withDictationSpacing,
} from './insertion'
describe('normalizeDictationText', () => {
it('trims and folds line breaks into single spaces', () => {
expect(normalizeDictationText(' hello \n\n world\r\n')).toBe('hello world')
})
it('reduces whitespace-only output to nothing', () => {
expect(normalizeDictationText(' \n \t ')).toBe('')
})
})
describe('withDictationSpacing', () => {
it('separates two latin words that would otherwise fuse', () => {
expect(withDictationSpacing('hello', 'big', ' world')).toBe(' big')
expect(withDictationSpacing('hello', 'big', 'world')).toBe(' big ')
})
it('leaves CJK text flush against its neighbours', () => {
expect(withDictationSpacing('你好', '世界', '再见')).toBe('世界')
})
it('does not double an existing space', () => {
expect(withDictationSpacing('hello ', 'big', ' world')).toBe('big')
})
it('adds a space after sentence punctuation before a latin word', () => {
expect(withDictationSpacing('Done.', 'Next', '')).toBe(' Next')
})
it('adds nothing at the edges of an empty draft', () => {
expect(withDictationSpacing('', 'hello', '')).toBe('hello')
})
})
describe('placeDictationResult', () => {
const base = { revisionAtStart: 3, revisionNow: 3, blocked: false, composing: false }
it('inserts when nothing changed', () => {
expect(placeDictationResult(base)).toBe('insert')
})
it('holds when the draft moved on, even if it was later restored', () => {
expect(placeDictationResult({ ...base, revisionNow: 5 })).toBe('hold')
})
it('holds while the composer is blocked by a send or a pending question', () => {
expect(placeDictationResult({ ...base, blocked: true })).toBe('hold')
})
it('holds during an IME composition even when nothing else changed', () => {
expect(placeDictationResult({ ...base, composing: true })).toBe('hold')
})
})
@@ -0,0 +1,45 @@
export type InsertionPoint = { start: number; end: number }
/** Recognizer output is a single line; collapse any line breaks it might carry. */
export function normalizeDictationText(raw: string): string {
return raw.replace(/\s*[\r\n]+\s*/g, ' ').trim()
}
const WORD_END = /[A-Za-z0-9.,;:!?)]$/
const WORD_START = /^[A-Za-z0-9(]/
/**
* Adds a space only where two alphanumeric runs would otherwise fuse. CJK
* output joins its neighbours directly, so it never gets one.
*/
export function withDictationSpacing(before: string, text: string, after: string): string {
const leading = WORD_END.test(before) && WORD_START.test(text) ? ' ' : ''
const trailing = WORD_END.test(text) && WORD_START.test(after) ? ' ' : ''
return `${leading}${text}${trailing}`
}
export type DictationResultPlacement =
/** Write at the position captured when recording began. */
| 'insert'
/** Keep the text and let the user decide. */
| 'hold'
/**
* Decides what to do with a transcript once it arrives.
*
* The draft is compared by revision, not by content: typing a character and
* deleting it again still moved the caret the captured position was measured
* against, and a message sent while the model was busy leaves an empty draft
* that looks identical to the one recording began in. An IME composition in
* progress counts as an edit in flight, so it holds too.
*/
export function placeDictationResult(input: {
revisionAtStart: number
revisionNow: number
blocked: boolean
composing: boolean
}): DictationResultPlacement {
if (input.composing || input.blocked || input.revisionNow !== input.revisionAtStart) return 'hold'
return 'insert'
}
@@ -0,0 +1,452 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { isVoiceCaptureSupported, startRecording, VoiceRecorderError } from './recorder'
class FakeTrack extends EventTarget {
stopped = false
stop() {
this.stopped = true
}
end() {
this.dispatchEvent(new Event('ended'))
}
}
class FakeStream {
tracks = [new FakeTrack()]
getTracks() {
return this.tracks
}
}
class FakeMediaRecorder extends EventTarget {
static instances: FakeMediaRecorder[] = []
static isTypeSupported = vi.fn((type: string) => type === 'audio/webm;codecs=opus')
state: 'inactive' | 'recording' = 'inactive'
mimeType: string
stopCalls = 0
constructor(public stream: FakeStream, options?: { mimeType?: string }) {
super()
this.mimeType = options?.mimeType ?? ''
FakeMediaRecorder.instances.push(this)
}
start() {
this.state = 'recording'
}
stop() {
this.stopCalls += 1
if (this.state === 'inactive') return
this.state = 'inactive'
this.dispatchEvent(Object.assign(new Event('dataavailable'), { data: new Blob(['audio-bytes']) }))
this.dispatchEvent(new Event('stop'))
}
}
let decodedSeconds = 2
let decodeShouldFail = false
const offlineConstructions: Array<[number, number, number]> = []
class FakeOfflineAudioContext {
destination = {}
constructor(public channels: number, public length: number, public rate: number) {
offlineConstructions.push([channels, length, rate])
}
async decodeAudioData() {
if (decodeShouldFail) throw new DOMException('bad data', 'EncodingError')
return {
sampleRate: 48_000,
numberOfChannels: 2,
duration: decodedSeconds,
getChannelData: () => new Float32Array(48_000 * decodedSeconds),
} as unknown as AudioBuffer
}
createBufferSource() {
return { buffer: null, connect: vi.fn(), start: vi.fn() }
}
async startRendering() {
return { getChannelData: () => new Float32Array(this.length).fill(0.1) } as unknown as AudioBuffer
}
}
const audioContexts: FakeAudioContext[] = []
class FakeAudioContext {
state = 'running'
closed = false
constructor() {
audioContexts.push(this)
}
createMediaStreamSource() {
return { connect: vi.fn(), disconnect: vi.fn() }
}
createAnalyser() {
return {
fftSize: 0,
getFloatTimeDomainData: (buffer: Float32Array) => buffer.fill(0.05),
}
}
async resume() {}
async close() {
this.closed = true
this.state = 'closed'
}
}
const originals = {
mediaDevices: Object.getOwnPropertyDescriptor(navigator, 'mediaDevices'),
secure: Object.getOwnPropertyDescriptor(window, 'isSecureContext'),
mediaRecorder: globalThis.MediaRecorder,
audioContext: globalThis.AudioContext,
offline: globalThis.OfflineAudioContext,
}
let getUserMedia: ReturnType<typeof vi.fn>
let streams: FakeStream[]
function domError(name: string) {
return new DOMException(name, name)
}
function grantMicrophone() {
getUserMedia.mockImplementation(async () => {
const stream = new FakeStream()
streams.push(stream)
return stream
})
}
function allTracksStopped() {
return streams.length > 0 && streams.every(stream => stream.tracks.every(track => track.stopped))
}
function setSecureContext(value: boolean) {
Object.defineProperty(window, 'isSecureContext', { value, configurable: true })
}
describe('recorder', () => {
beforeEach(() => {
streams = []
audioContexts.length = 0
offlineConstructions.length = 0
FakeMediaRecorder.instances = []
decodedSeconds = 2
decodeShouldFail = false
getUserMedia = vi.fn()
Object.defineProperty(navigator, 'mediaDevices', { value: { getUserMedia }, configurable: true })
setSecureContext(true)
globalThis.MediaRecorder = FakeMediaRecorder as unknown as typeof MediaRecorder
globalThis.AudioContext = FakeAudioContext as unknown as typeof AudioContext
globalThis.OfflineAudioContext = FakeOfflineAudioContext as unknown as typeof OfflineAudioContext
grantMicrophone()
})
afterEach(() => {
vi.useRealTimers()
if (originals.mediaDevices) Object.defineProperty(navigator, 'mediaDevices', originals.mediaDevices)
else delete (navigator as { mediaDevices?: unknown }).mediaDevices
if (originals.secure) Object.defineProperty(window, 'isSecureContext', originals.secure)
else delete (window as { isSecureContext?: unknown }).isSecureContext
globalThis.MediaRecorder = originals.mediaRecorder
globalThis.AudioContext = originals.audioContext
globalThis.OfflineAudioContext = originals.offline
})
describe('isVoiceCaptureSupported', () => {
it('is true with a secure context, getUserMedia and MediaRecorder', () => {
expect(isVoiceCaptureSupported()).toBe(true)
})
it('is false outside a secure context (plain-http LAN pages)', () => {
setSecureContext(false)
expect(isVoiceCaptureSupported()).toBe(false)
})
it('is false without getUserMedia', () => {
Object.defineProperty(navigator, 'mediaDevices', { value: {}, configurable: true })
expect(isVoiceCaptureSupported()).toBe(false)
})
it('is false without MediaRecorder', () => {
globalThis.MediaRecorder = undefined as unknown as typeof MediaRecorder
expect(isVoiceCaptureSupported()).toBe(false)
})
})
describe('a successful recording', () => {
it('returns a 16 kHz mono WAV and releases every track', async () => {
decodedSeconds = 1.5
const recording = await startRecording({ maxSeconds: 60 })
const result = await recording.stop()
expect(result.seconds).toBeCloseTo(1.5, 5)
expect(result.wav.type).toBe('audio/wav')
// 44-byte header plus 24000 16-bit samples.
expect(result.wav.size).toBe(44 + 24_000 * 2)
// decodeAudioData context first, then the 48 kHz stereo -> 16 kHz mono render.
expect(offlineConstructions).toEqual([[1, 1, 16_000], [1, 24_000, 16_000]])
expect(allTracksStopped()).toBe(true)
expect(audioContexts.every(context => context.closed)).toBe(true)
})
it('asks for echo cancellation and noise suppression on the default device', async () => {
const recording = await startRecording({ maxSeconds: 60 })
expect(getUserMedia).toHaveBeenCalledWith({ audio: { echoCancellation: true, noiseSuppression: true } })
recording.cancel()
})
it('pins the preferred device with an exact constraint', async () => {
const recording = await startRecording({ maxSeconds: 60, deviceId: 'usb-1' })
expect(getUserMedia).toHaveBeenCalledWith({
audio: { echoCancellation: true, noiseSuppression: true, deviceId: { exact: 'usb-1' } },
})
recording.cancel()
})
it('prefers an opus container when the browser supports one', async () => {
const recording = await startRecording({ maxSeconds: 60 })
expect(FakeMediaRecorder.instances[0]!.mimeType).toBe('audio/webm;codecs=opus')
recording.cancel()
})
it('shares one result between repeated stop calls', async () => {
const recording = await startRecording({ maxSeconds: 60 })
const [first, second] = await Promise.all([recording.stop(), recording.stop()])
expect(second).toBe(first)
expect(FakeMediaRecorder.instances).toHaveLength(1)
})
it('reports input loudness while recording and zero afterwards', async () => {
const recording = await startRecording({ maxSeconds: 60 })
expect(recording.getLevel()).toBeGreaterThan(0)
expect(recording.getLevel()).toBeLessThanOrEqual(1)
recording.cancel()
expect(recording.getLevel()).toBe(0)
})
})
describe('device fallback', () => {
it.each(['OverconstrainedError', 'NotFoundError'])(
'records from the default device when the remembered one raises %s',
async (name) => {
getUserMedia.mockRejectedValueOnce(domError(name))
const recording = await startRecording({ maxSeconds: 60, deviceId: 'gone' })
expect(getUserMedia).toHaveBeenCalledTimes(2)
expect(getUserMedia).toHaveBeenLastCalledWith({ audio: { echoCancellation: true, noiseSuppression: true } })
recording.cancel()
expect(allTracksStopped()).toBe(true)
},
)
it('reports no-device when even the default device is missing', async () => {
getUserMedia.mockRejectedValue(domError('NotFoundError'))
await expect(startRecording({ maxSeconds: 60, deviceId: 'gone' })).rejects.toMatchObject({ code: 'no-device' })
expect(getUserMedia).toHaveBeenCalledTimes(2)
})
it('does not fall back on a permission error', async () => {
getUserMedia.mockRejectedValue(domError('NotAllowedError'))
await expect(startRecording({ maxSeconds: 60, deviceId: 'usb-1' })).rejects.toMatchObject({ code: 'permission' })
expect(getUserMedia).toHaveBeenCalledTimes(1)
})
})
describe('capture errors', () => {
it.each([
['NotAllowedError', 'permission'],
['SecurityError', 'permission'],
['NotFoundError', 'no-device'],
['NotReadableError', 'device-busy'],
['TrackStartError', 'device-busy'],
['TypeError', 'failed'],
])('maps %s to %s', async (name, code) => {
getUserMedia.mockRejectedValue(domError(name))
const error = await startRecording({ maxSeconds: 60 }).catch(reason => reason)
expect(error).toBeInstanceOf(VoiceRecorderError)
expect(error.code).toBe(code)
})
it('reports unavailable without touching the microphone when the API is missing', async () => {
setSecureContext(false)
await expect(startRecording({ maxSeconds: 60 })).rejects.toMatchObject({ code: 'unavailable' })
expect(getUserMedia).not.toHaveBeenCalled()
})
it('releases the stream when the recorder cannot be constructed', async () => {
globalThis.MediaRecorder = class {
static isTypeSupported = () => false
constructor() {
throw domError('NotSupportedError')
}
} as unknown as typeof MediaRecorder
await expect(startRecording({ maxSeconds: 60 })).rejects.toMatchObject({ code: 'failed' })
expect(allTracksStopped()).toBe(true)
})
})
describe('cancellation', () => {
it('never opens the microphone when already aborted', async () => {
const controller = new AbortController()
controller.abort()
await expect(startRecording({ maxSeconds: 60, signal: controller.signal })).rejects.toMatchObject({ name: 'AbortError' })
expect(getUserMedia).not.toHaveBeenCalled()
})
it('stops the stream at once when permission arrives after the abort', async () => {
const controller = new AbortController()
let grant!: (stream: FakeStream) => void
getUserMedia.mockImplementation(() => new Promise(resolve => { grant = resolve as typeof grant }))
const pending = startRecording({ maxSeconds: 60, signal: controller.signal })
const rejection = expect(pending).rejects.toMatchObject({ name: 'AbortError' })
controller.abort()
const stream = new FakeStream()
streams.push(stream)
grant(stream)
await rejection
expect(stream.tracks[0]!.stopped).toBe(true)
expect(FakeMediaRecorder.instances).toHaveLength(0)
})
it('discards the recording and stops every track', async () => {
const recording = await startRecording({ maxSeconds: 60 })
recording.cancel()
expect(allTracksStopped()).toBe(true)
expect(audioContexts.every(context => context.closed)).toBe(true)
expect(FakeMediaRecorder.instances[0]!.state).toBe('inactive')
})
it('is idempotent', async () => {
const recording = await startRecording({ maxSeconds: 60 })
recording.cancel()
const stopCalls = FakeMediaRecorder.instances[0]!.stopCalls
expect(() => recording.cancel()).not.toThrow()
expect(FakeMediaRecorder.instances[0]!.stopCalls).toBe(stopCalls)
})
it('rejects stop() after a cancel instead of producing audio', async () => {
const recording = await startRecording({ maxSeconds: 60 })
recording.cancel()
await expect(recording.stop()).rejects.toBeInstanceOf(VoiceRecorderError)
})
it('drops the result when cancelled while the recording is being decoded', async () => {
const recording = await startRecording({ maxSeconds: 60 })
const result = recording.stop()
recording.cancel()
await expect(result).rejects.toBeInstanceOf(VoiceRecorderError)
})
})
describe('maxSeconds', () => {
it('stops capturing and notifies once the cap is reached', async () => {
vi.useFakeTimers()
const onLimitReached = vi.fn()
const recording = await startRecording({ maxSeconds: 3, onLimitReached })
vi.advanceTimersByTime(2999)
expect(onLimitReached).not.toHaveBeenCalled()
vi.advanceTimersByTime(1)
expect(onLimitReached).toHaveBeenCalledTimes(1)
expect(FakeMediaRecorder.instances[0]!.state).toBe('inactive')
vi.useRealTimers()
decodedSeconds = 3
await expect(recording.stop()).resolves.toMatchObject({ seconds: 3 })
})
it('truncates audio that overshoots the cap', async () => {
decodedSeconds = 10
const recording = await startRecording({ maxSeconds: 3 })
const result = await recording.stop()
expect(result.seconds).toBe(3)
expect(result.wav.size).toBe(44 + 3 * 16_000 * 2)
})
it('does not fire the limit callback after a manual stop', async () => {
vi.useFakeTimers()
const onLimitReached = vi.fn()
const recording = await startRecording({ maxSeconds: 3, onLimitReached })
const result = recording.stop()
await vi.runAllTimersAsync()
await result
expect(onLimitReached).not.toHaveBeenCalled()
})
})
describe('interruption', () => {
it('reports a track that ends mid-recording and rejects stop()', async () => {
const onInterrupted = vi.fn()
const recording = await startRecording({ maxSeconds: 60, onInterrupted })
streams[0]!.tracks[0]!.end()
expect(onInterrupted).toHaveBeenCalledTimes(1)
expect(onInterrupted.mock.calls[0]![0]).toMatchObject({ code: 'interrupted' })
expect(allTracksStopped()).toBe(true)
await expect(recording.stop()).rejects.toMatchObject({ code: 'interrupted' })
})
it('reports a recorder error as a failure', async () => {
const onInterrupted = vi.fn()
const recording = await startRecording({ maxSeconds: 60, onInterrupted })
FakeMediaRecorder.instances[0]!.dispatchEvent(new Event('error'))
expect(onInterrupted.mock.calls[0]![0]).toMatchObject({ code: 'failed' })
await expect(recording.stop()).rejects.toMatchObject({ code: 'failed' })
})
it('does not treat its own cleanup as an interruption', async () => {
const onInterrupted = vi.fn()
const recording = await startRecording({ maxSeconds: 60, onInterrupted })
await recording.stop()
streams[0]!.tracks[0]!.end()
expect(onInterrupted).not.toHaveBeenCalled()
})
})
describe('decoding', () => {
it('turns a decode failure on a real recording into a failed error after the microphone is already off', async () => {
vi.useFakeTimers({ toFake: ['Date'] })
decodeShouldFail = true
const recording = await startRecording({ maxSeconds: 60 })
vi.setSystemTime(Date.now() + 2000)
await expect(recording.stop()).rejects.toMatchObject({ code: 'failed' })
expect(allTracksStopped()).toBe(true)
})
// Chromium cannot decode the container MediaRecorder writes for a click
// shorter than about 100 ms. That is a mis-tap, not a broken decoder.
it('reports a click too short to decode as an empty recording, not as a failure', async () => {
vi.useFakeTimers({ toFake: ['Date'] })
decodeShouldFail = true
const recording = await startRecording({ maxSeconds: 60 })
vi.setSystemTime(Date.now() + 30)
const result = await recording.stop()
expect(result.seconds).toBe(0)
expect(result.wav.size).toBe(44)
expect(allTracksStopped()).toBe(true)
})
it('still decodes a short click normally when the browser can', async () => {
vi.useFakeTimers({ toFake: ['Date'] })
decodedSeconds = 0.2
const recording = await startRecording({ maxSeconds: 60 })
vi.setSystemTime(Date.now() + 200)
await expect(recording.stop()).resolves.toMatchObject({ seconds: expect.closeTo(0.2, 5) })
})
})
})
+310
View File
@@ -0,0 +1,310 @@
import { encodeWav, toMono16k, VOICE_SAMPLE_RATE } from './wav'
export type VoiceRecorderErrorCode =
| 'unavailable'
| 'permission'
| 'no-device'
| 'device-busy'
| 'interrupted'
| 'failed'
export class VoiceRecorderError extends Error {
readonly code: VoiceRecorderErrorCode
constructor(code: VoiceRecorderErrorCode, message?: string) {
super(message ?? code)
this.name = 'VoiceRecorderError'
this.code = code
}
}
export type RecordingResult = { wav: Blob; seconds: number }
export interface ActiveRecording {
/** Input loudness in 0..1. Cheap enough to poll from requestAnimationFrame. */
getLevel(): number
/**
* Ends the recording and resolves with the 16 kHz mono WAV. Repeated calls
* share one result. Rejects with a `VoiceRecorderError` when the capture was
* interrupted or cancelled.
*/
stop(): Promise<RecordingResult>
/** Discards the recording and releases the microphone. Idempotent. */
cancel(): void
}
export type StartRecordingOptions = {
/** Undefined, or an id that no longer exists, records from the system default. */
deviceId?: string
maxSeconds: number
/**
* The cap was reached and the recorder has already stopped capturing. The
* caller still owns the microphone until it calls `stop()` or `cancel()`.
*/
onLimitReached?: () => void
/** The device vanished or the recorder failed while recording. */
onInterrupted?: (error: VoiceRecorderError) => void
/** Aborting before the microphone opens releases it the moment it arrives. */
signal?: AbortSignal
}
const MIME_CANDIDATES = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4', 'audio/ogg;codecs=opus']
const RECORDER_TIMESLICE_MS = 250
const LEVEL_GAIN = 5
/**
* MediaRecorder writes a container Chromium cannot decode when it is stopped
* within roughly 100 ms of starting. A decode failure inside this window is a
* mis-tap, so it is reported as an empty recording rather than a failure.
*/
const UNDECODABLE_CLICK_MS = 500
export function isVoiceCaptureSupported(): boolean {
if (typeof window === 'undefined' || typeof navigator === 'undefined') return false
return Boolean(
window.isSecureContext &&
typeof navigator.mediaDevices?.getUserMedia === 'function' &&
typeof MediaRecorder !== 'undefined',
)
}
function errorName(error: unknown): string {
return typeof error === 'object' && error !== null && 'name' in error ? String((error as { name: unknown }).name) : ''
}
function mapCaptureError(error: unknown): VoiceRecorderError {
if (error instanceof VoiceRecorderError) return error
const message = error instanceof Error ? error.message : undefined
switch (errorName(error)) {
case 'NotAllowedError':
case 'SecurityError':
case 'PermissionDeniedError':
return new VoiceRecorderError('permission', message)
case 'NotFoundError':
case 'DevicesNotFoundError':
return new VoiceRecorderError('no-device', message)
case 'NotReadableError':
case 'TrackStartError':
return new VoiceRecorderError('device-busy', message)
default:
return new VoiceRecorderError('failed', message)
}
}
function abortError(): DOMException {
return new DOMException('Recording was cancelled', 'AbortError')
}
function stopStream(stream: MediaStream) {
for (const track of stream.getTracks()) {
try {
track.stop()
} catch {
// A track that is already gone has nothing left to release.
}
}
}
async function acquireStream(deviceId: string | undefined): Promise<MediaStream> {
const audio = { echoCancellation: true, noiseSuppression: true }
const { getUserMedia } = navigator.mediaDevices
if (!deviceId) return getUserMedia.call(navigator.mediaDevices, { audio })
try {
return await getUserMedia.call(navigator.mediaDevices, { audio: { ...audio, deviceId: { exact: deviceId } } })
} catch (error) {
// A remembered microphone that was unplugged is not an error for the user;
// record from whatever the system offers now.
const name = errorName(error)
if (name !== 'OverconstrainedError' && name !== 'NotFoundError') throw error
return getUserMedia.call(navigator.mediaDevices, { audio })
}
}
async function readBlob(blob: Blob): Promise<ArrayBuffer> {
if (typeof blob.arrayBuffer === 'function') return blob.arrayBuffer()
return new Promise((resolve, reject) => {
const reader = new FileReader()
reader.onload = () => resolve(reader.result as ArrayBuffer)
reader.onerror = () => reject(reader.error)
reader.readAsArrayBuffer(blob)
})
}
function createDeferred() {
let resolve!: () => void
const promise = new Promise<void>(done => { resolve = done })
return { promise, resolve }
}
export async function startRecording(options: StartRecordingOptions): Promise<ActiveRecording> {
if (!isVoiceCaptureSupported()) throw new VoiceRecorderError('unavailable')
if (options.signal?.aborted) throw abortError()
let stream: MediaStream
try {
stream = await acquireStream(options.deviceId)
} catch (error) {
throw mapCaptureError(error)
}
// Permission prompts outlive the click that caused them. If the caller gave
// up while the prompt was open, the stream must not stay live behind them.
if (options.signal?.aborted) {
stopStream(stream)
throw abortError()
}
let recorder: MediaRecorder
try {
const mimeType = typeof MediaRecorder.isTypeSupported === 'function'
? MIME_CANDIDATES.find(candidate => MediaRecorder.isTypeSupported(candidate))
: undefined
recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined)
} catch (error) {
stopStream(stream)
throw mapCaptureError(error)
}
const tracks = stream.getTracks()
const chunks: Blob[] = []
const recorderStopped = createDeferred()
let cancelled = false
let released = false
let failure: VoiceRecorderError | null = null
let stopPromise: Promise<RecordingResult> | null = null
let startedAt = 0
let limitTimer: ReturnType<typeof setTimeout> | undefined
let audioContext: AudioContext | undefined
let source: MediaStreamAudioSourceNode | undefined
let analyser: AnalyserNode | undefined
let levelBuffer: Float32Array<ArrayBuffer> | undefined
const onData = (event: Event) => {
const data = (event as BlobEvent).data
if (data && data.size > 0) chunks.push(data)
}
const onRecorderStop = () => recorderStopped.resolve()
const onRecorderError = () => interrupt(new VoiceRecorderError('failed', 'The recorder reported an error'))
const onTrackEnded = () => interrupt(new VoiceRecorderError('interrupted', 'The microphone stopped delivering audio'))
const release = () => {
if (released) return
released = true
clearTimeout(limitTimer)
recorder.removeEventListener('dataavailable', onData)
recorder.removeEventListener('stop', onRecorderStop)
recorder.removeEventListener('error', onRecorderError)
for (const track of tracks) track.removeEventListener('ended', onTrackEnded)
if (recorder.state !== 'inactive') {
try {
recorder.stop()
} catch {
// Already stopping.
}
}
stopStream(stream)
try {
source?.disconnect()
} catch {
// Not connected.
}
if (audioContext && audioContext.state !== 'closed') void audioContext.close().catch(() => {})
}
function interrupt(error: VoiceRecorderError) {
if (released || cancelled || failure) return
failure = error
release()
recorderStopped.resolve()
options.onInterrupted?.(error)
}
recorder.addEventListener('dataavailable', onData)
recorder.addEventListener('stop', onRecorderStop)
recorder.addEventListener('error', onRecorderError)
for (const track of tracks) track.addEventListener('ended', onTrackEnded)
try {
// The level meter is a nicety; a browser without it still records.
audioContext = new AudioContext()
source = audioContext.createMediaStreamSource(stream)
analyser = audioContext.createAnalyser()
analyser.fftSize = 1024
source.connect(analyser)
levelBuffer = new Float32Array(analyser.fftSize)
void audioContext.resume?.()?.catch(() => {})
} catch {
analyser = undefined
}
try {
recorder.start(RECORDER_TIMESLICE_MS)
startedAt = Date.now()
} catch (error) {
release()
throw mapCaptureError(error)
}
limitTimer = setTimeout(() => {
if (released || cancelled || failure) return
if (recorder.state !== 'inactive') {
try {
recorder.stop()
} catch {
// Already stopping.
}
}
options.onLimitReached?.()
}, options.maxSeconds * 1000)
const finalize = async (): Promise<RecordingResult> => {
const elapsedMs = Date.now() - startedAt
if (!cancelled && !failure && recorder.state !== 'inactive') {
try {
recorder.stop()
} catch (error) {
interrupt(mapCaptureError(error))
}
}
await recorderStopped.promise
if (cancelled) throw new VoiceRecorderError('failed', 'Recording was cancelled')
if (failure) throw failure
const blob = new Blob(chunks, { type: recorder.mimeType || chunks[0]?.type || '' })
// Turn the microphone off before the slow part.
release()
let samples: Float32Array
try {
const decoder = new OfflineAudioContext(1, 1, VOICE_SAMPLE_RATE)
const decoded = await decoder.decodeAudioData(await readBlob(blob))
samples = await toMono16k(decoded)
} catch (error) {
if (elapsedMs < UNDECODABLE_CLICK_MS) return { wav: encodeWav(new Float32Array(0)), seconds: 0 }
throw new VoiceRecorderError('failed', error instanceof Error ? error.message : 'Could not decode the recording')
}
if (cancelled) throw new VoiceRecorderError('failed', 'Recording was cancelled')
const maxSamples = Math.floor(options.maxSeconds * VOICE_SAMPLE_RATE)
if (samples.length > maxSamples) samples = samples.subarray(0, maxSamples)
return { wav: encodeWav(samples), seconds: samples.length / VOICE_SAMPLE_RATE }
}
return {
getLevel() {
if (!analyser || !levelBuffer || released) return 0
analyser.getFloatTimeDomainData(levelBuffer)
let sum = 0
for (let i = 0; i < levelBuffer.length; i += 1) sum += levelBuffer[i]! * levelBuffer[i]!
return Math.min(1, Math.sqrt(sum / levelBuffer.length) * LEVEL_GAIN)
},
stop() {
stopPromise ??= finalize()
return stopPromise
},
cancel() {
if (cancelled) return
cancelled = true
release()
recorderStopped.resolve()
},
}
}
@@ -0,0 +1,359 @@
import { useCallback, useEffect, useMemo, useRef, useState, type RefObject } from 'react'
import { ApiError } from '@/api/client'
import { voiceApi } from '@/api/voice'
import { useVoiceInputStore } from '@/stores/voiceInputStore'
import type { MentionComposerHandle } from '@/components/chat/MentionComposer'
import { getPreferredMicrophoneId } from './devicePreference'
import {
normalizeDictationText,
placeDictationResult,
withDictationSpacing,
type InsertionPoint,
} from './insertion'
import {
startRecording,
VoiceRecorderError,
type ActiveRecording,
type VoiceRecorderErrorCode,
} from './recorder'
export type DictationPhase = 'idle' | 'starting' | 'recording' | 'transcribing'
/** Doubles as the suffix of the `voice.composer.error.*` translation keys. */
export type DictationIssue =
| 'permission'
| 'noDevice'
| 'deviceBusy'
| 'unavailable'
| 'interrupted'
| 'notReady'
| 'invalidAudio'
| 'unknownProvider'
| 'failed'
| 'noSpeech'
| 'tooShort'
type Run = {
controller: AbortController
recording: ActiveRecording | null
revisionAtStart: number
point: InsertionPoint
stopping: boolean
}
type ComposerDictationOptions = {
composerRef: RefObject<MentionComposerHandle | null>
/** The composer's current projected text; every distinct value is one revision. */
draft: string
/** The composer cannot take text right now (disabled, or a message is being sent). */
blocked: boolean
/** Identifies what the composer is editing; a change abandons any dictation. */
contextKey: string | null | undefined
}
/** Recordings shorter than this are a mis-tap, not speech. */
const MIN_SECONDS = 0.4
const ISSUE_TIMEOUT_MS = 8_000
function issueFromRecorder(code: VoiceRecorderErrorCode): DictationIssue {
switch (code) {
case 'permission': return 'permission'
case 'no-device': return 'noDevice'
case 'device-busy': return 'deviceBusy'
case 'unavailable': return 'unavailable'
case 'interrupted': return 'interrupted'
default: return 'failed'
}
}
function issueFromError(error: unknown): DictationIssue {
if (error instanceof VoiceRecorderError) return issueFromRecorder(error.code)
if (error instanceof ApiError) {
const code = typeof error.body === 'object' && error.body !== null && 'error' in error.body
? (error.body as { error: unknown }).error
: undefined
if (code === 'voice/not-ready') return 'notReady'
if (code === 'voice/invalid-audio') return 'invalidAudio'
if (code === 'voice/unknown-provider') return 'unknownProvider'
if (code === 'voice/failed') return 'failed'
if (error.status === 409) return 'notReady'
if (error.status === 400) return 'invalidAudio'
if (error.status === 404) return 'unknownProvider'
}
return 'failed'
}
/**
* Dictation for a ProseMirror composer: record, transcribe, and write the text
* into the draft without ever overwriting what the user did meanwhile.
*
* Shared by both composers (ChatInput and EmptySession) so the write-back rules
* cannot drift between them.
*/
export function useComposerDictation({ composerRef, draft, blocked, contextKey }: ComposerDictationOptions) {
const providerId = useVoiceInputStore(state => state.catalog?.preferences.providerId)
const language = useVoiceInputStore(state => state.catalog?.preferences.language)
const maxSeconds = useVoiceInputStore(state => state.catalog?.limits.maxAudioSeconds)
const [phase, setPhase] = useState<DictationPhase>('idle')
const [issue, setIssue] = useState<DictationIssue | null>(null)
const [pendingText, setPendingText] = useState<string | null>(null)
const [startedAt, setStartedAt] = useState(0)
const activeRef = useRef<Run | null>(null)
const draftRef = useRef(draft)
const revisionRef = useRef(0)
const blockedRef = useRef(blocked)
const settingsRef = useRef({ providerId, language, maxSeconds })
const composingRef = useRef(false)
const pendingRef = useRef<string | null>(null)
const previousContextRef = useRef(contextKey)
blockedRef.current = blocked
settingsRef.current = { providerId, language, maxSeconds }
useEffect(() => {
if (draftRef.current !== draft) {
draftRef.current = draft
revisionRef.current += 1
}
}, [draft])
const setPending = useCallback((text: string | null) => {
pendingRef.current = text
setPendingText(text)
}, [])
const capturePoint = useCallback((): InsertionPoint => {
const composer = composerRef.current
if (composer?.hasFocus()) {
const { start, end } = composer.getSelectionOffsets()
return { start, end }
}
const length = draftRef.current.length
return { start: length, end: length }
}, [composerRef])
const writeText = useCallback((text: string, point: InsertionPoint) => {
const composer = composerRef.current
if (!composer) return
const current = draftRef.current
const start = Math.min(point.start, current.length)
const end = Math.min(Math.max(point.end, start), current.length)
composer.insertTextAtOffsets(start, end, withDictationSpacing(current.slice(0, start), text, current.slice(end)))
}, [composerRef])
const cancel = useCallback(() => {
const run = activeRef.current
activeRef.current = null
if (run) {
run.controller.abort()
run.recording?.cancel()
}
setPhase('idle')
}, [])
const abandon = useCallback(() => {
cancel()
setPending(null)
setIssue(null)
}, [cancel, setPending])
const deliver = useCallback((run: Run, rawText: string) => {
const text = normalizeDictationText(rawText)
if (!text) {
setIssue('noSpeech')
return
}
const placement = placeDictationResult({
revisionAtStart: run.revisionAtStart,
revisionNow: revisionRef.current,
blocked: blockedRef.current,
composing: composingRef.current,
})
if (placement === 'insert') {
writeText(text, run.point)
return
}
setPending(text)
}, [setPending, writeText])
const finish = useCallback(async (run: Run) => {
if (activeRef.current !== run || !run.recording || run.stopping) return
run.stopping = true
setPhase('transcribing')
const fail = (next: DictationIssue) => {
activeRef.current = null
setPhase('idle')
setIssue(next)
}
try {
const { wav, seconds } = await run.recording.stop()
if (activeRef.current !== run) return
if (seconds < MIN_SECONDS) {
fail('tooShort')
return
}
const { providerId: provider, language: lang } = settingsRef.current
if (!provider || !lang) {
fail('notReady')
return
}
const transcript = await voiceApi.transcribe(wav, {
providerId: provider,
language: lang,
signal: run.controller.signal,
})
if (activeRef.current !== run) return
activeRef.current = null
setPhase('idle')
deliver(run, transcript.text)
} catch (error) {
if (activeRef.current !== run) return
const next = issueFromError(error)
// The server disagrees with our cached catalog (the model was deleted
// behind our back); refresh it so the button stops being offered.
if (next === 'notReady') void useVoiceInputStore.getState().loadCatalog({ force: true })
fail(next)
}
}, [deliver])
const start = useCallback(async () => {
const settings = settingsRef.current
if (activeRef.current || !settings.providerId || !settings.maxSeconds) return
const run: Run = {
controller: new AbortController(),
recording: null,
revisionAtStart: revisionRef.current,
point: capturePoint(),
stopping: false,
}
activeRef.current = run
// Starting over is a deliberate choice; the previous held text goes.
setPending(null)
setIssue(null)
setPhase('starting')
let recording: ActiveRecording
try {
recording = await startRecording({
deviceId: getPreferredMicrophoneId(),
maxSeconds: settings.maxSeconds,
signal: run.controller.signal,
onLimitReached: () => { void finish(run) },
onInterrupted: (error) => {
if (activeRef.current !== run) return
activeRef.current = null
setPhase('idle')
setIssue(issueFromRecorder(error.code))
},
})
} catch (error) {
if (activeRef.current !== run) return
activeRef.current = null
setPhase('idle')
setIssue(issueFromError(error))
return
}
// Cancelled while the permission prompt was open, or another run took over.
if (activeRef.current !== run) {
recording.cancel()
return
}
run.recording = recording
setStartedAt(Date.now())
setPhase('recording')
}, [capturePoint, finish, setPending])
const toggle = useCallback(() => {
const run = activeRef.current
if (!run) {
void start()
} else if (run.recording) {
void finish(run)
} else {
cancel()
}
}, [cancel, finish, start])
const insertPending = useCallback(() => {
const text = pendingRef.current
if (text === null || blockedRef.current) return
writeText(text, capturePoint())
setPending(null)
composerRef.current?.focus()
}, [capturePoint, composerRef, setPending, writeText])
const dismissPending = useCallback(() => setPending(null), [setPending])
const dismissIssue = useCallback(() => setIssue(null), [])
const onCompositionStart = useCallback(() => {
composingRef.current = true
}, [])
const onCompositionEnd = useCallback(() => {
composingRef.current = false
}, [])
// Esc backs out of a recording or an in-flight recognition. Captured ahead of
// the composer so it does not also close a menu or interrupt a turn.
useEffect(() => {
if (phase === 'idle') return
const onKeyDown = (event: KeyboardEvent) => {
if (event.key !== 'Escape' || event.isComposing) return
event.preventDefault()
event.stopPropagation()
cancel()
}
window.addEventListener('keydown', onKeyDown, true)
return () => window.removeEventListener('keydown', onKeyDown, true)
}, [cancel, phase])
useEffect(() => {
if (!issue) return
const timer = setTimeout(() => setIssue(null), ISSUE_TIMEOUT_MS)
return () => clearTimeout(timer)
}, [issue])
// A different session means a different draft: nothing recorded or held for
// the old one belongs here.
useEffect(() => {
if (previousContextRef.current === contextKey) return
previousContextRef.current = contextKey
abandon()
}, [abandon, contextKey])
useEffect(() => abandon, [abandon])
const getLevel = useCallback(() => activeRef.current?.recording?.getLevel() ?? 0, [])
return useMemo(() => ({
phase,
issue,
pendingText,
startedAt,
toggle,
cancel,
insertPending,
dismissPending,
dismissIssue,
getLevel,
compositionHandlers: { onCompositionStart, onCompositionEnd },
}), [
phase,
issue,
pendingText,
startedAt,
toggle,
cancel,
insertPending,
dismissPending,
dismissIssue,
getLevel,
onCompositionStart,
onCompositionEnd,
])
}
export type ComposerDictation = ReturnType<typeof useComposerDictation>
+107
View File
@@ -0,0 +1,107 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { encodeWav, encodeWavBytes, floatToPcm16, toMono16k, VOICE_SAMPLE_RATE } from './wav'
function ascii(view: DataView, offset: number, length: number) {
return String.fromCharCode(...Array.from({ length }, (_, i) => view.getUint8(offset + i)))
}
describe('encodeWavBytes', () => {
it('writes a canonical 44-byte mono PCM16 header', () => {
const bytes = encodeWavBytes(new Float32Array(160))
const view = new DataView(bytes.buffer)
expect(bytes.byteLength).toBe(44 + 160 * 2)
expect(ascii(view, 0, 4)).toBe('RIFF')
expect(view.getUint32(4, true)).toBe(36 + 160 * 2)
expect(ascii(view, 8, 4)).toBe('WAVE')
expect(ascii(view, 12, 4)).toBe('fmt ')
expect(view.getUint32(16, true)).toBe(16)
expect(view.getUint16(20, true)).toBe(1) // PCM
expect(view.getUint16(22, true)).toBe(1) // mono
expect(view.getUint32(24, true)).toBe(16_000)
expect(view.getUint32(28, true)).toBe(32_000)
expect(view.getUint16(32, true)).toBe(2)
expect(view.getUint16(34, true)).toBe(16)
expect(ascii(view, 36, 4)).toBe('data')
expect(view.getUint32(40, true)).toBe(160 * 2)
})
it('stores samples little-endian and clamps out-of-range input', () => {
const bytes = encodeWavBytes(new Float32Array([0, 1, -1, 0.5, 2, -2]))
const view = new DataView(bytes.buffer)
const read = (index: number) => view.getInt16(44 + index * 2, true)
expect([0, 1, 2, 3, 4, 5].map(read)).toEqual([0, 32767, -32768, 16384, 32767, -32768])
})
it('records the requested sample rate in the header', () => {
const view = new DataView(encodeWavBytes(new Float32Array(1), 8000).buffer)
expect(view.getUint32(24, true)).toBe(8000)
expect(view.getUint32(28, true)).toBe(16_000)
})
it('wraps the bytes in an audio/wav blob', () => {
const blob = encodeWav(new Float32Array(10))
expect(blob.type).toBe('audio/wav')
expect(blob.size).toBe(44 + 20)
})
})
describe('floatToPcm16', () => {
it('maps silence to zero', () => {
expect(Array.from(floatToPcm16(new Float32Array(3)))).toEqual([0, 0, 0])
})
})
describe('toMono16k', () => {
const originalOffline = globalThis.OfflineAudioContext
afterEach(() => {
globalThis.OfflineAudioContext = originalOffline
})
function fakeBuffer(sampleRate: number, channels: number, seconds: number) {
const length = Math.round(sampleRate * seconds)
return {
sampleRate,
numberOfChannels: channels,
duration: seconds,
length,
getChannelData: () => new Float32Array(length).fill(0.25),
} as unknown as AudioBuffer
}
it('returns a buffer that already matches without rendering', async () => {
const offline = vi.fn()
globalThis.OfflineAudioContext = offline as unknown as typeof OfflineAudioContext
const samples = await toMono16k(fakeBuffer(VOICE_SAMPLE_RATE, 1, 0.5))
expect(samples).toHaveLength(8000)
expect(offline).not.toHaveBeenCalled()
})
it('renders other rates and channel counts through a 16 kHz mono offline context', async () => {
const constructed: Array<[number, number, number]> = []
class FakeOffline {
destination = {}
constructor(channels: number, length: number, rate: number) {
constructed.push([channels, length, rate])
}
createBufferSource() {
return { buffer: null as AudioBuffer | null, connect: vi.fn(), start: vi.fn() }
}
async startRendering() {
const length = constructed[0]![1]
return { getChannelData: () => new Float32Array(length) } as unknown as AudioBuffer
}
}
globalThis.OfflineAudioContext = FakeOffline as unknown as typeof OfflineAudioContext
// 1.5 s at 48 kHz stereo must come out as exactly 1.5 s at 16 kHz.
const samples = await toMono16k(fakeBuffer(48_000, 2, 1.5))
expect(constructed).toEqual([[1, 24_000, 16_000]])
expect(samples).toHaveLength(24_000)
})
})
+71
View File
@@ -0,0 +1,71 @@
/** The only format the voice service accepts: 16 kHz, mono, 16-bit PCM. */
export const VOICE_SAMPLE_RATE = 16_000
const WAV_HEADER_BYTES = 44
function writeAscii(view: DataView, offset: number, text: string) {
for (let i = 0; i < text.length; i += 1) view.setUint8(offset + i, text.charCodeAt(i))
}
/** Float samples in [-1, 1] to little-endian PCM16, clamping anything outside. */
export function floatToPcm16(samples: Float32Array): Int16Array {
const out = new Int16Array(samples.length)
for (let i = 0; i < samples.length; i += 1) {
const clamped = Math.max(-1, Math.min(1, samples[i]!))
// Asymmetric scale keeps +1.0 at 32767 and -1.0 at -32768.
out[i] = clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff)
}
return out
}
/** A canonical 44-byte-header RIFF/WAVE file holding mono PCM16. */
export function encodeWavBytes(samples: Float32Array, sampleRate: number = VOICE_SAMPLE_RATE): Uint8Array {
const pcm = floatToPcm16(samples)
const dataBytes = pcm.length * 2
const buffer = new ArrayBuffer(WAV_HEADER_BYTES + dataBytes)
const view = new DataView(buffer)
writeAscii(view, 0, 'RIFF')
view.setUint32(4, 36 + dataBytes, true)
writeAscii(view, 8, 'WAVE')
writeAscii(view, 12, 'fmt ')
view.setUint32(16, 16, true) // fmt chunk size
view.setUint16(20, 1, true) // PCM
view.setUint16(22, 1, true) // mono
view.setUint32(24, sampleRate, true)
view.setUint32(28, sampleRate * 2, true) // byte rate
view.setUint16(32, 2, true) // block align
view.setUint16(34, 16, true) // bits per sample
writeAscii(view, 36, 'data')
view.setUint32(40, dataBytes, true)
// Explicit little-endian writes: an Int16Array view would follow the host's
// byte order.
for (let i = 0; i < pcm.length; i += 1) view.setInt16(WAV_HEADER_BYTES + i * 2, pcm[i]!, true)
return new Uint8Array(buffer)
}
export function encodeWav(samples: Float32Array, sampleRate: number = VOICE_SAMPLE_RATE): Blob {
return new Blob([encodeWavBytes(samples, sampleRate) as BlobPart], { type: 'audio/wav' })
}
/**
* Mixes a decoded buffer down to mono and resamples it to 16 kHz.
*
* The offline context does both: a one-channel destination downmixes on connect
* and the destination rate drives the resample, so no hand-written filter is
* needed. Buffers that already match skip the render.
*/
export async function toMono16k(buffer: AudioBuffer): Promise<Float32Array> {
if (buffer.sampleRate === VOICE_SAMPLE_RATE && buffer.numberOfChannels === 1) {
return buffer.getChannelData(0).slice()
}
const length = Math.max(1, Math.ceil(buffer.duration * VOICE_SAMPLE_RATE))
const offline = new OfflineAudioContext(1, length, VOICE_SAMPLE_RATE)
const source = offline.createBufferSource()
source.buffer = buffer
source.connect(offline.destination)
source.start(0)
const rendered = await offline.startRendering()
return rendered.getChannelData(0).slice()
}
+98
View File
@@ -3913,6 +3913,104 @@ Row 9, all 8 cells: continuing from straight down, turning left through lower-le
'workflows.history.agents': '{count} agents recorded',
'workflows.history.script': 'Script',
'workflows.launcher.argsPlaceholder': 'e.g. a question, or a list of paths',
'settings.tab.voice': 'Voice Input',
'voice.settings.title': 'Voice Input',
'voice.settings.description': 'Dictate into the chat box with a speech model that runs on the machine hosting the cc-haha server. Audio is never sent to a cloud service.',
'voice.settings.loadFailed': 'Could not load voice input settings.',
'voice.settings.saveFailed': 'Could not save this setting. Please try again.',
'voice.settings.actionFailed': 'That action did not complete.',
'voice.settings.unsupported.title': 'Local speech recognition is not supported on this platform yet',
'voice.settings.unsupported.detail': 'Voice input needs a platform that the local speech runtime supports.',
'voice.settings.enable.label': 'Enable voice input',
'voice.settings.enable.description': 'Show a dictation button next to the chat box. Recorded audio is recognized on the machine that runs the cc-haha server.',
'voice.settings.enable.needModel': 'Voice input is on, but the speech model is not downloaded yet. Download it below; nothing is downloaded automatically.',
'voice.settings.engine.title': 'Recognition engine',
'voice.settings.engine.description': 'Choose the speech model and manage its files.',
'voice.settings.engine.provider': 'Engine',
'voice.settings.phase.unprepared': 'Not downloaded',
'voice.settings.phase.downloading': 'Downloading',
'voice.settings.phase.verifying': 'Verifying',
'voice.settings.phase.ready': 'Ready',
'voice.settings.phase.failed': 'Failed',
'voice.settings.phase.cancelled': 'Paused',
'voice.settings.step.runtime': 'Downloading speech runtime',
'voice.settings.step.model': 'Downloading speech model',
'voice.settings.step.vad': 'Downloading voice activity model',
'voice.settings.step.verify': 'Verifying files',
'voice.settings.model.unprepared': 'The speech model is not downloaded.',
'voice.settings.model.download': 'Download',
'voice.settings.model.meta': 'Local recognition · about {size}',
'voice.settings.model.downloading': 'Downloading',
'voice.settings.model.progress': '{completed} / {total}',
'voice.settings.model.resumed': 'Resumed from {size} that was already downloaded',
'voice.settings.model.source': 'Source: {source}',
'voice.settings.model.verifying': 'Verifying downloaded files…',
'voice.settings.model.ready': 'Local model is ready. It wakes up automatically when you record.',
'voice.settings.model.remove': 'Delete model',
'voice.settings.model.removeTitle': 'Delete the local speech model?',
'voice.settings.model.removeBody': 'This removes the downloaded runtime and model files. You will need to download them again before using voice input.',
'voice.settings.model.cancelled': 'Paused. Parts already downloaded are kept, so you can continue any time.',
'voice.settings.model.resume': 'Continue download',
'voice.settings.model.resumeHint': 'Parts already downloaded are kept. Retry once the network is back to continue where it stopped.',
'voice.settings.failure.network': 'Download failed: network connection problem',
'voice.settings.failure.dns': 'Download failed: could not resolve the download address',
'voice.settings.failure.timeout': 'Download failed: the connection timed out',
'voice.settings.failure.certificate': 'Download failed: the secure connection could not be verified',
'voice.settings.failure.http': 'Download failed: the server returned an error',
'voice.settings.failure.integrity': 'Download failed: the file check did not match, so the file will be downloaded again',
'voice.settings.failure.storage': 'Download failed: could not write to disk (check free space and permissions)',
'voice.settings.failure.unsupportedPlatform': 'This platform is not supported by the local speech runtime',
'voice.settings.failure.unknown': 'Download failed for an unknown reason',
'voice.settings.language.label': 'Language',
'voice.settings.language.auto': 'Auto detect',
'voice.settings.language.zh': 'Chinese (Mandarin)',
'voice.settings.language.en': 'English',
'voice.settings.language.ja': 'Japanese',
'voice.settings.language.ko': 'Korean',
'voice.settings.language.yue': 'Cantonese',
'voice.settings.mic.title': 'Microphone',
'voice.settings.mic.description': 'Choose which input device is used for dictation. This choice is saved on this device only.',
'voice.settings.mic.label': 'Input device',
'voice.settings.mic.systemDefault': 'System default',
'voice.settings.mic.savedDevice': 'Saved microphone',
'voice.settings.mic.unnamed': 'Microphone {index}',
'voice.settings.mic.missing': 'The microphone you chose earlier is not available. The system default is used instead.',
'voice.settings.mic.noDevices': 'No microphone was detected.',
'voice.settings.mic.allowAccess': 'Allow microphone access to show device names',
'voice.settings.mic.allowHint': 'The microphone opens briefly to unlock device names. Nothing is recorded.',
'voice.settings.capture.unsupported': 'The microphone is not available in this context. Open cc-haha on this computer or over HTTPS to record.',
'voice.settings.test.title': 'Transcription test',
'voice.settings.test.description': 'Record a short clip and see what the model recognizes, using the same path as dictation.',
'voice.settings.test.start': 'Start test',
'voice.settings.test.stop': 'Stop',
'voice.settings.test.starting': 'Starting…',
'voice.settings.test.transcribing': 'Recognizing…',
'voice.settings.test.level': 'Microphone level',
'voice.settings.test.needModel': 'Download the speech model above before testing.',
'voice.settings.test.result': 'Recognized text',
'voice.settings.test.stats': 'Audio {audio} s · Inference {inference} s',
'voice.settings.test.playback': 'Play back the recording',
'voice.composer.start': 'Dictate',
'voice.composer.starting': 'Opening microphone… (click to cancel)',
'voice.composer.stop': 'Stop recording and transcribe',
'voice.composer.transcribing': 'Transcribing…',
'voice.composer.recordingHint': 'Recording — press Esc to cancel',
'voice.composer.insertText': 'Insert text',
'voice.composer.discard': 'Discard',
'voice.composer.dismiss': 'Dismiss',
'voice.composer.pendingTitle': 'Dictated text',
'voice.composer.pendingHint': 'The draft changed while transcribing, so this was not added automatically.',
'voice.composer.error.permission': 'Microphone access was denied. Allow it in your system settings and try again.',
'voice.composer.error.noDevice': 'No microphone was found. Connect one and try again.',
'voice.composer.error.deviceBusy': 'The microphone is in use by another app.',
'voice.composer.error.unavailable': 'Dictation is not available in this environment.',
'voice.composer.error.interrupted': 'The microphone stopped unexpectedly. Recording was discarded.',
'voice.composer.error.notReady': 'The speech model is not ready. Check Settings → Voice input.',
'voice.composer.error.invalidAudio': 'The recording could not be read. Try again.',
'voice.composer.error.unknownProvider': 'The selected speech engine is unavailable.',
'voice.composer.error.failed': 'Transcription failed. Try again.',
'voice.composer.error.noSpeech': 'No speech was recognized.',
'voice.composer.error.tooShort': 'The recording was too short.',
} as const
export type TranslationKey = keyof typeof en
+98
View File
@@ -3914,4 +3914,102 @@ export const jp: Record<TranslationKey, string> = {
'workflows.history.agents': '{count} 個のエージェントを記録',
'workflows.history.script': 'スクリプト',
'workflows.launcher.argsPlaceholder': '例: 質問、またはパスのリスト',
'settings.tab.voice': '音声入力',
'voice.settings.title': '音声入力',
'voice.settings.description': 'cc-haha サーバーを実行しているこのマシン上のローカル音声モデルで、話した内容を入力欄に書き起こします。音声がクラウドサービスに送信されることはありません。',
'voice.settings.loadFailed': '音声入力の設定を読み込めませんでした。',
'voice.settings.saveFailed': '設定を保存できませんでした。もう一度お試しください。',
'voice.settings.actionFailed': '操作を完了できませんでした。',
'voice.settings.unsupported.title': 'このプラットフォームではローカル音声認識はまだ利用できません',
'voice.settings.unsupported.detail': '音声入力にはローカル音声ランタイムが対応するプラットフォームが必要です。',
'voice.settings.enable.label': '音声入力を有効にする',
'voice.settings.enable.description': 'チャット入力欄の横にディクテーションボタンを表示します。録音した音声は cc-haha サーバーを実行しているマシン上で認識されます。',
'voice.settings.enable.needModel': '音声入力は有効ですが、音声モデルがまだダウンロードされていません。下でダウンロードしてください。自動ではダウンロードされません。',
'voice.settings.engine.title': '認識エンジン',
'voice.settings.engine.description': '音声モデルを選び、そのファイルを管理します。',
'voice.settings.engine.provider': 'エンジン',
'voice.settings.phase.unprepared': '未ダウンロード',
'voice.settings.phase.downloading': 'ダウンロード中',
'voice.settings.phase.verifying': '検証中',
'voice.settings.phase.ready': '準備完了',
'voice.settings.phase.failed': '失敗',
'voice.settings.phase.cancelled': '一時停止',
'voice.settings.step.runtime': '音声ランタイムをダウンロード中',
'voice.settings.step.model': '音声モデルをダウンロード中',
'voice.settings.step.vad': '音声区間検出モデルをダウンロード中',
'voice.settings.step.verify': 'ファイルを検証中',
'voice.settings.model.unprepared': '音声モデルはまだダウンロードされていません。',
'voice.settings.model.download': 'ダウンロード',
'voice.settings.model.meta': 'ローカル認識 · 約 {size}',
'voice.settings.model.downloading': 'ダウンロード中',
'voice.settings.model.progress': '{completed} / {total}',
'voice.settings.model.resumed': 'ダウンロード済みの {size} から再開しました',
'voice.settings.model.source': 'ダウンロード元: {source}',
'voice.settings.model.verifying': 'ダウンロードしたファイルを検証しています…',
'voice.settings.model.ready': 'ローカルモデルの準備ができました。録音時に自動で起動します。',
'voice.settings.model.remove': 'モデルを削除',
'voice.settings.model.removeTitle': 'ローカル音声モデルを削除しますか?',
'voice.settings.model.removeBody': 'ダウンロード済みのランタイムとモデルファイルを削除します。音声入力を使うには再度ダウンロードが必要です。',
'voice.settings.model.cancelled': '一時停止中です。ダウンロード済みの部分は保持されるので、いつでも再開できます。',
'voice.settings.model.resume': 'ダウンロードを再開',
'voice.settings.model.resumeHint': 'ダウンロード済みの部分は保持されます。ネットワークが回復したら再試行すると、中断した位置から続けられます。',
'voice.settings.failure.network': 'ダウンロード失敗: ネットワーク接続の問題',
'voice.settings.failure.dns': 'ダウンロード失敗: ダウンロード先のアドレスを解決できません',
'voice.settings.failure.timeout': 'ダウンロード失敗: 接続がタイムアウトしました',
'voice.settings.failure.certificate': 'ダウンロード失敗: 安全な接続を確認できません',
'voice.settings.failure.http': 'ダウンロード失敗: サーバーがエラーを返しました',
'voice.settings.failure.integrity': 'ダウンロード失敗: ファイルの検証が一致しなかったため、再ダウンロードします',
'voice.settings.failure.storage': 'ダウンロード失敗: ディスクに書き込めません(空き容量と権限を確認してください)',
'voice.settings.failure.unsupportedPlatform': 'このプラットフォームはローカル音声ランタイムに対応していません',
'voice.settings.failure.unknown': 'ダウンロードに失敗しました(原因不明)',
'voice.settings.language.label': '言語',
'voice.settings.language.auto': '自動検出',
'voice.settings.language.zh': '中国語(標準語)',
'voice.settings.language.en': '英語',
'voice.settings.language.ja': '日本語',
'voice.settings.language.ko': '韓国語',
'voice.settings.language.yue': '広東語',
'voice.settings.mic.title': 'マイク',
'voice.settings.mic.description': 'ディクテーションに使う入力デバイスを選びます。この選択はこのデバイスにのみ保存されます。',
'voice.settings.mic.label': '入力デバイス',
'voice.settings.mic.systemDefault': 'システムのデフォルト',
'voice.settings.mic.savedDevice': '保存したマイク',
'voice.settings.mic.unnamed': 'マイク {index}',
'voice.settings.mic.missing': '以前選んだマイクは現在利用できないため、システムのデフォルトを使用します。',
'voice.settings.mic.noDevices': 'マイクが検出されませんでした。',
'voice.settings.mic.allowAccess': 'マイクへのアクセスを許可してデバイス名を表示',
'voice.settings.mic.allowHint': 'デバイス名を表示するためにマイクを一瞬だけ開きます。録音はされません。',
'voice.settings.capture.unsupported': 'この環境ではマイクを使用できません。このコンピューターで cc-haha を開くか、HTTPS 経由で録音してください。',
'voice.settings.test.title': '文字起こしテスト',
'voice.settings.test.description': '短い音声を録音し、ディクテーションと同じ経路でモデルの認識結果を確認します。',
'voice.settings.test.start': 'テスト開始',
'voice.settings.test.stop': '停止',
'voice.settings.test.starting': '開始中…',
'voice.settings.test.transcribing': '認識中…',
'voice.settings.test.level': 'マイクレベル',
'voice.settings.test.needModel': 'テストの前に、上で音声モデルをダウンロードしてください。',
'voice.settings.test.result': '認識結果',
'voice.settings.test.stats': '音声 {audio} 秒 · 推論 {inference} 秒',
'voice.settings.test.playback': '録音を再生',
'voice.composer.start': '音声入力',
'voice.composer.starting': 'マイクを開いています…(クリックでキャンセル)',
'voice.composer.stop': '録音を停止して認識',
'voice.composer.transcribing': '認識中…',
'voice.composer.recordingHint': '録音中 · Esc でキャンセル',
'voice.composer.insertText': 'テキストを挿入',
'voice.composer.discard': '破棄',
'voice.composer.dismiss': '閉じる',
'voice.composer.pendingTitle': '音声認識の結果',
'voice.composer.pendingHint': '認識中に入力欄の内容が変わったため、自動では挿入しませんでした。',
'voice.composer.error.permission': 'マイクへのアクセスが拒否されました。システム設定で許可してからもう一度お試しください。',
'voice.composer.error.noDevice': 'マイクが見つかりません。接続してからもう一度お試しください。',
'voice.composer.error.deviceBusy': 'マイクは別のアプリで使用中です。',
'voice.composer.error.unavailable': 'この環境では音声入力を利用できません。',
'voice.composer.error.interrupted': 'マイクが予期せず停止しました。録音は破棄されました。',
'voice.composer.error.notReady': '音声モデルの準備ができていません。設定 → 音声入力を確認してください。',
'voice.composer.error.invalidAudio': '録音を読み取れませんでした。もう一度お試しください。',
'voice.composer.error.unknownProvider': '選択した音声エンジンは利用できません。',
'voice.composer.error.failed': '認識に失敗しました。もう一度お試しください。',
'voice.composer.error.noSpeech': '音声が認識されませんでした。',
'voice.composer.error.tooShort': '録音が短すぎます。',
}
+98
View File
@@ -3916,4 +3916,102 @@ export const kr: Record<TranslationKey, string> = {
'workflows.history.agents': '{count}개 에이전트 기록됨',
'workflows.history.script': '스크립트',
'workflows.launcher.argsPlaceholder': '예: 질문 또는 경로 목록',
'settings.tab.voice': '음성 입력',
'voice.settings.title': '음성 입력',
'voice.settings.description': 'cc-haha 서버를 실행 중인 이 컴퓨터의 로컬 음성 모델로 말한 내용을 입력창에 받아 적습니다. 오디오는 클라우드 서비스로 전송되지 않습니다.',
'voice.settings.loadFailed': '음성 입력 설정을 불러오지 못했습니다.',
'voice.settings.saveFailed': '설정을 저장하지 못했습니다. 다시 시도해 주세요.',
'voice.settings.actionFailed': '작업을 완료하지 못했습니다.',
'voice.settings.unsupported.title': '이 플랫폼에서는 아직 로컬 음성 인식을 지원하지 않습니다',
'voice.settings.unsupported.detail': '음성 입력에는 로컬 음성 런타임이 지원하는 플랫폼이 필요합니다.',
'voice.settings.enable.label': '음성 입력 사용',
'voice.settings.enable.description': '채팅 입력창 옆에 받아쓰기 버튼을 표시합니다. 녹음한 오디오는 cc-haha 서버를 실행 중인 컴퓨터에서 인식됩니다.',
'voice.settings.enable.needModel': '음성 입력이 켜져 있지만 음성 모델이 아직 다운로드되지 않았습니다. 아래에서 다운로드하세요. 자동으로 다운로드되지 않습니다.',
'voice.settings.engine.title': '인식 엔진',
'voice.settings.engine.description': '음성 모델을 선택하고 파일을 관리합니다.',
'voice.settings.engine.provider': '엔진',
'voice.settings.phase.unprepared': '다운로드 안 됨',
'voice.settings.phase.downloading': '다운로드 중',
'voice.settings.phase.verifying': '검증 중',
'voice.settings.phase.ready': '준비됨',
'voice.settings.phase.failed': '실패',
'voice.settings.phase.cancelled': '일시 중지됨',
'voice.settings.step.runtime': '음성 런타임 다운로드 중',
'voice.settings.step.model': '음성 모델 다운로드 중',
'voice.settings.step.vad': '음성 활동 감지 모델 다운로드 중',
'voice.settings.step.verify': '파일 검증 중',
'voice.settings.model.unprepared': '음성 모델이 아직 다운로드되지 않았습니다.',
'voice.settings.model.download': '다운로드',
'voice.settings.model.meta': '로컬 인식 · 약 {size}',
'voice.settings.model.downloading': '다운로드 중',
'voice.settings.model.progress': '{completed} / {total}',
'voice.settings.model.resumed': '이미 받은 {size}부터 이어받았습니다',
'voice.settings.model.source': '다운로드 출처: {source}',
'voice.settings.model.verifying': '다운로드한 파일을 검증하는 중…',
'voice.settings.model.ready': '로컬 모델이 준비되었습니다. 녹음할 때 자동으로 시작됩니다.',
'voice.settings.model.remove': '모델 삭제',
'voice.settings.model.removeTitle': '로컬 음성 모델을 삭제할까요?',
'voice.settings.model.removeBody': '다운로드한 런타임과 모델 파일을 삭제합니다. 음성 입력을 사용하려면 다시 다운로드해야 합니다.',
'voice.settings.model.cancelled': '일시 중지되었습니다. 이미 받은 부분은 보존되므로 언제든 이어서 받을 수 있습니다.',
'voice.settings.model.resume': '다운로드 이어받기',
'voice.settings.model.resumeHint': '이미 받은 부분은 보존됩니다. 네트워크가 복구되면 다시 시도하여 중단된 지점부터 이어받으세요.',
'voice.settings.failure.network': '다운로드 실패: 네트워크 연결 문제',
'voice.settings.failure.dns': '다운로드 실패: 다운로드 주소를 확인할 수 없습니다',
'voice.settings.failure.timeout': '다운로드 실패: 연결 시간이 초과되었습니다',
'voice.settings.failure.certificate': '다운로드 실패: 보안 연결을 확인할 수 없습니다',
'voice.settings.failure.http': '다운로드 실패: 서버가 오류를 반환했습니다',
'voice.settings.failure.integrity': '다운로드 실패: 파일 검증이 일치하지 않아 파일을 다시 받습니다',
'voice.settings.failure.storage': '다운로드 실패: 디스크에 쓸 수 없습니다(여유 공간과 권한을 확인하세요)',
'voice.settings.failure.unsupportedPlatform': '로컬 음성 런타임이 이 플랫폼을 지원하지 않습니다',
'voice.settings.failure.unknown': '알 수 없는 이유로 다운로드에 실패했습니다',
'voice.settings.language.label': '언어',
'voice.settings.language.auto': '자동 감지',
'voice.settings.language.zh': '중국어(표준어)',
'voice.settings.language.en': '영어',
'voice.settings.language.ja': '일본어',
'voice.settings.language.ko': '한국어',
'voice.settings.language.yue': '광둥어',
'voice.settings.mic.title': '마이크',
'voice.settings.mic.description': '받아쓰기에 사용할 입력 장치를 선택합니다. 이 선택은 이 기기에만 저장됩니다.',
'voice.settings.mic.label': '입력 장치',
'voice.settings.mic.systemDefault': '시스템 기본값',
'voice.settings.mic.savedDevice': '저장된 마이크',
'voice.settings.mic.unnamed': '마이크 {index}',
'voice.settings.mic.missing': '이전에 선택한 마이크를 지금은 사용할 수 없어 시스템 기본값을 사용합니다.',
'voice.settings.mic.noDevices': '마이크가 감지되지 않았습니다.',
'voice.settings.mic.allowAccess': '마이크 접근을 허용하여 장치 이름 표시',
'voice.settings.mic.allowHint': '장치 이름을 표시하기 위해 마이크를 잠깐 열 뿐이며 녹음하지 않습니다.',
'voice.settings.capture.unsupported': '이 환경에서는 마이크를 사용할 수 없습니다. 이 컴퓨터에서 cc-haha를 열거나 HTTPS로 접속해 녹음하세요.',
'voice.settings.test.title': '받아쓰기 테스트',
'voice.settings.test.description': '짧게 녹음하고 받아쓰기와 같은 경로로 모델이 인식한 결과를 확인합니다.',
'voice.settings.test.start': '테스트 시작',
'voice.settings.test.stop': '중지',
'voice.settings.test.starting': '시작하는 중…',
'voice.settings.test.transcribing': '인식 중…',
'voice.settings.test.level': '마이크 레벨',
'voice.settings.test.needModel': '테스트 전에 위에서 음성 모델을 먼저 다운로드하세요.',
'voice.settings.test.result': '인식 결과',
'voice.settings.test.stats': '오디오 {audio}초 · 추론 {inference}초',
'voice.settings.test.playback': '녹음 재생',
'voice.composer.start': '음성 입력',
'voice.composer.starting': '마이크를 여는 중… (클릭하면 취소)',
'voice.composer.stop': '녹음을 멈추고 인식',
'voice.composer.transcribing': '인식 중…',
'voice.composer.recordingHint': '녹음 중 · Esc로 취소',
'voice.composer.insertText': '텍스트 삽입',
'voice.composer.discard': '버리기',
'voice.composer.dismiss': '닫기',
'voice.composer.pendingTitle': '음성 인식 결과',
'voice.composer.pendingHint': '인식 중에 입력란 내용이 바뀌어 자동으로 넣지 않았습니다.',
'voice.composer.error.permission': '마이크 접근이 거부되었습니다. 시스템 설정에서 허용한 뒤 다시 시도하세요.',
'voice.composer.error.noDevice': '마이크를 찾을 수 없습니다. 연결한 뒤 다시 시도하세요.',
'voice.composer.error.deviceBusy': '다른 앱이 마이크를 사용 중입니다.',
'voice.composer.error.unavailable': '이 환경에서는 음성 입력을 사용할 수 없습니다.',
'voice.composer.error.interrupted': '마이크가 예기치 않게 중단되어 녹음이 폐기되었습니다.',
'voice.composer.error.notReady': '음성 모델이 준비되지 않았습니다. 설정 → 음성 입력을 확인하세요.',
'voice.composer.error.invalidAudio': '녹음을 읽을 수 없습니다. 다시 시도하세요.',
'voice.composer.error.unknownProvider': '선택한 음성 엔진을 사용할 수 없습니다.',
'voice.composer.error.failed': '인식에 실패했습니다. 다시 시도해 주세요.',
'voice.composer.error.noSpeech': '음성이 인식되지 않았습니다.',
'voice.composer.error.tooShort': '녹음이 너무 짧습니다.',
}
+98
View File
@@ -3913,4 +3913,102 @@ export const zh: Record<TranslationKey, string> = {
'workflows.history.agents': '已記錄 {count} 個 agent',
'workflows.history.script': '腳本',
'workflows.launcher.argsPlaceholder': '例如一個問題,或一組路徑',
'settings.tab.voice': '語音輸入',
'voice.settings.title': '語音輸入',
'voice.settings.description': '使用執行 cc-haha 服務的這台電腦上的本機語音模型,把說話內容聽寫到輸入框。音訊不會傳送到任何雲端服務。',
'voice.settings.loadFailed': '無法載入語音輸入設定。',
'voice.settings.saveFailed': '儲存設定失敗,請重試。',
'voice.settings.actionFailed': '操作未完成。',
'voice.settings.unsupported.title': '目前平台暫不支援本機語音辨識',
'voice.settings.unsupported.detail': '語音輸入需要本機語音執行環境支援的平台。',
'voice.settings.enable.label': '啟用語音輸入',
'voice.settings.enable.description': '在聊天輸入框旁顯示聽寫按鈕。錄下的音訊會在執行 cc-haha 服務的這台電腦上辨識。',
'voice.settings.enable.needModel': '語音輸入已開啟,但語音模型尚未下載。請在下方下載;不會自動下載。',
'voice.settings.engine.title': '辨識引擎',
'voice.settings.engine.description': '選擇語音模型並管理其檔案。',
'voice.settings.engine.provider': '引擎',
'voice.settings.phase.unprepared': '未下載',
'voice.settings.phase.downloading': '下載中',
'voice.settings.phase.verifying': '驗證中',
'voice.settings.phase.ready': '已就緒',
'voice.settings.phase.failed': '失敗',
'voice.settings.phase.cancelled': '已暫停',
'voice.settings.step.runtime': '正在下載語音執行環境',
'voice.settings.step.model': '正在下載語音模型',
'voice.settings.step.vad': '正在下載語音活動偵測模型',
'voice.settings.step.verify': '正在驗證檔案',
'voice.settings.model.unprepared': '語音模型尚未下載。',
'voice.settings.model.download': '下載',
'voice.settings.model.meta': '本機辨識 · 約 {size}',
'voice.settings.model.downloading': '下載中',
'voice.settings.model.progress': '{completed} / {total}',
'voice.settings.model.resumed': '已從斷點續傳,已下載 {size}',
'voice.settings.model.source': '下載來源:{source}',
'voice.settings.model.verifying': '正在驗證已下載的檔案…',
'voice.settings.model.ready': '本機模型已就緒,錄音時自動喚醒。',
'voice.settings.model.remove': '刪除模型',
'voice.settings.model.removeTitle': '刪除本機語音模型?',
'voice.settings.model.removeBody': '將刪除已下載的執行環境和模型檔案。之後需要重新下載才能使用語音輸入。',
'voice.settings.model.cancelled': '已暫停,已下載的部分會保留,可隨時繼續。',
'voice.settings.model.resume': '繼續下載',
'voice.settings.model.resumeHint': '已下載的部分會保留。網路恢復後點選重試即可從中斷處繼續。',
'voice.settings.failure.network': '下載失敗:網路連線異常',
'voice.settings.failure.dns': '下載失敗:無法解析下載位址',
'voice.settings.failure.timeout': '下載失敗:連線逾時',
'voice.settings.failure.certificate': '下載失敗:無法驗證安全連線',
'voice.settings.failure.http': '下載失敗:伺服器回傳了錯誤',
'voice.settings.failure.integrity': '下載失敗:檔案驗證不一致,將重新下載該檔案',
'voice.settings.failure.storage': '下載失敗:無法寫入磁碟(請檢查剩餘空間和權限)',
'voice.settings.failure.unsupportedPlatform': '本機語音執行環境不支援目前平台',
'voice.settings.failure.unknown': '下載失敗,原因不明',
'voice.settings.language.label': '語言',
'voice.settings.language.auto': '自動偵測',
'voice.settings.language.zh': '中文(普通話)',
'voice.settings.language.en': '英語',
'voice.settings.language.ja': '日語',
'voice.settings.language.ko': '韓語',
'voice.settings.language.yue': '粵語',
'voice.settings.mic.title': '麥克風',
'voice.settings.mic.description': '選擇聽寫使用的輸入裝置。此選擇只會儲存在這台裝置上。',
'voice.settings.mic.label': '輸入裝置',
'voice.settings.mic.systemDefault': '系統預設',
'voice.settings.mic.savedDevice': '已儲存的麥克風',
'voice.settings.mic.unnamed': '麥克風 {index}',
'voice.settings.mic.missing': '先前選擇的麥克風目前無法使用,已改用系統預設麥克風。',
'voice.settings.mic.noDevices': '沒有偵測到麥克風。',
'voice.settings.mic.allowAccess': '允許存取麥克風以顯示裝置名稱',
'voice.settings.mic.allowHint': '會短暫開啟麥克風以顯示裝置名稱,不會錄音。',
'voice.settings.capture.unsupported': '目前環境無法使用麥克風。請在本機開啟 cc-haha,或透過 HTTPS 存取後再錄音。',
'voice.settings.test.title': '轉錄測試',
'voice.settings.test.description': '錄一小段音,用與聽寫相同的路徑查看模型辨識出的文字。',
'voice.settings.test.start': '開始測試',
'voice.settings.test.stop': '停止',
'voice.settings.test.starting': '正在啟動…',
'voice.settings.test.transcribing': '辨識中…',
'voice.settings.test.level': '麥克風音量',
'voice.settings.test.needModel': '請先在上方下載語音模型,再進行測試。',
'voice.settings.test.result': '辨識結果',
'voice.settings.test.stats': '音訊 {audio} 秒 · 推論 {inference} 秒',
'voice.settings.test.playback': '重播錄音',
'voice.composer.start': '語音輸入',
'voice.composer.starting': '正在開啟麥克風…(點擊取消)',
'voice.composer.stop': '停止錄音並辨識',
'voice.composer.transcribing': '辨識中…',
'voice.composer.recordingHint': '錄音中,按 Esc 取消',
'voice.composer.insertText': '插入文字',
'voice.composer.discard': '捨棄',
'voice.composer.dismiss': '關閉提示',
'voice.composer.pendingTitle': '語音辨識結果',
'voice.composer.pendingHint': '辨識期間輸入框內容已變更,未自動寫入。',
'voice.composer.error.permission': '麥克風權限被拒絕,請在系統設定中允許後重試。',
'voice.composer.error.noDevice': '未偵測到麥克風,請連接後重試。',
'voice.composer.error.deviceBusy': '麥克風正被其他應用程式佔用。',
'voice.composer.error.unavailable': '目前環境不支援語音輸入。',
'voice.composer.error.interrupted': '麥克風意外中斷,本次錄音已捨棄。',
'voice.composer.error.notReady': '語音模型尚未就緒,請在「設定 → 語音輸入」中檢查。',
'voice.composer.error.invalidAudio': '無法讀取錄音,請重試。',
'voice.composer.error.unknownProvider': '所選的語音引擎無法使用。',
'voice.composer.error.failed': '辨識失敗,請重試。',
'voice.composer.error.noSpeech': '未辨識到語音。',
'voice.composer.error.tooShort': '錄音時間太短。',
}
+98
View File
@@ -3912,4 +3912,102 @@ export const zh: Record<TranslationKey, string> = {
'workflows.history.agents': '已记录 {count} 个 agent',
'workflows.history.script': '脚本',
'workflows.launcher.argsPlaceholder': '例如一个问题,或一组路径',
'settings.tab.voice': '语音输入',
'voice.settings.title': '语音输入',
'voice.settings.description': '用运行 cc-haha 服务的这台电脑上的本地语音模型,把说话内容听写到输入框。音频不会发送到任何云端服务。',
'voice.settings.loadFailed': '无法加载语音输入设置。',
'voice.settings.saveFailed': '保存设置失败,请重试。',
'voice.settings.actionFailed': '操作没有完成。',
'voice.settings.unsupported.title': '当前平台暂不支持本地语音识别',
'voice.settings.unsupported.detail': '语音输入需要本地语音运行时支持的平台。',
'voice.settings.enable.label': '启用语音输入',
'voice.settings.enable.description': '在聊天输入框旁显示听写按钮。录下的音频在运行 cc-haha 服务的这台电脑上识别。',
'voice.settings.enable.needModel': '语音输入已开启,但语音模型还没有下载。请在下方下载;不会自动下载。',
'voice.settings.engine.title': '识别引擎',
'voice.settings.engine.description': '选择语音模型并管理其文件。',
'voice.settings.engine.provider': '引擎',
'voice.settings.phase.unprepared': '未下载',
'voice.settings.phase.downloading': '下载中',
'voice.settings.phase.verifying': '校验中',
'voice.settings.phase.ready': '已就绪',
'voice.settings.phase.failed': '失败',
'voice.settings.phase.cancelled': '已暂停',
'voice.settings.step.runtime': '正在下载语音运行时',
'voice.settings.step.model': '正在下载语音模型',
'voice.settings.step.vad': '正在下载语音活动检测模型',
'voice.settings.step.verify': '正在校验文件',
'voice.settings.model.unprepared': '语音模型尚未下载。',
'voice.settings.model.download': '下载',
'voice.settings.model.meta': '本地识别 · 约 {size}',
'voice.settings.model.downloading': '下载中',
'voice.settings.model.progress': '{completed} / {total}',
'voice.settings.model.resumed': '已从断点续传,已下载 {size}',
'voice.settings.model.source': '下载源:{source}',
'voice.settings.model.verifying': '正在校验已下载的文件…',
'voice.settings.model.ready': '本地模型已就绪,录音时自动唤醒。',
'voice.settings.model.remove': '删除模型',
'voice.settings.model.removeTitle': '删除本地语音模型?',
'voice.settings.model.removeBody': '将删除已下载的运行时和模型文件。之后需要重新下载才能使用语音输入。',
'voice.settings.model.cancelled': '已暂停,已下载的部分会保留,可随时继续。',
'voice.settings.model.resume': '继续下载',
'voice.settings.model.resumeHint': '已下载的部分会保留。网络恢复后点击重试即可从中断处继续。',
'voice.settings.failure.network': '下载失败:网络连接异常',
'voice.settings.failure.dns': '下载失败:无法解析下载地址',
'voice.settings.failure.timeout': '下载失败:连接超时',
'voice.settings.failure.certificate': '下载失败:无法验证安全连接',
'voice.settings.failure.http': '下载失败:服务器返回了错误',
'voice.settings.failure.integrity': '下载失败:文件校验不一致,将重新下载该文件',
'voice.settings.failure.storage': '下载失败:无法写入磁盘(请检查剩余空间和权限)',
'voice.settings.failure.unsupportedPlatform': '本地语音运行时不支持当前平台',
'voice.settings.failure.unknown': '下载失败,原因未知',
'voice.settings.language.label': '语言',
'voice.settings.language.auto': '自动检测',
'voice.settings.language.zh': '中文(普通话)',
'voice.settings.language.en': '英语',
'voice.settings.language.ja': '日语',
'voice.settings.language.ko': '韩语',
'voice.settings.language.yue': '粤语',
'voice.settings.mic.title': '麦克风',
'voice.settings.mic.description': '选择听写使用的输入设备。该选择只保存在这台设备上。',
'voice.settings.mic.label': '输入设备',
'voice.settings.mic.systemDefault': '系统默认',
'voice.settings.mic.savedDevice': '已保存的麦克风',
'voice.settings.mic.unnamed': '麦克风 {index}',
'voice.settings.mic.missing': '之前选择的麦克风当前不可用,已改用系统默认麦克风。',
'voice.settings.mic.noDevices': '没有检测到麦克风。',
'voice.settings.mic.allowAccess': '允许访问麦克风以显示设备名称',
'voice.settings.mic.allowHint': '会短暂打开麦克风以显示设备名称,不会录音。',
'voice.settings.capture.unsupported': '当前环境无法使用麦克风。请在本机打开 cc-haha,或通过 HTTPS 访问后再录音。',
'voice.settings.test.title': '转录测试',
'voice.settings.test.description': '录一小段音,用与听写相同的链路查看模型识别出的文字。',
'voice.settings.test.start': '开始测试',
'voice.settings.test.stop': '停止',
'voice.settings.test.starting': '正在启动…',
'voice.settings.test.transcribing': '识别中…',
'voice.settings.test.level': '麦克风电平',
'voice.settings.test.needModel': '请先在上方下载语音模型,再进行测试。',
'voice.settings.test.result': '识别结果',
'voice.settings.test.stats': '音频 {audio} 秒 · 推理 {inference} 秒',
'voice.settings.test.playback': '回放录音',
'voice.composer.start': '语音输入',
'voice.composer.starting': '正在打开麦克风…(点击取消)',
'voice.composer.stop': '停止录音并识别',
'voice.composer.transcribing': '识别中…',
'voice.composer.recordingHint': '录音中,按 Esc 取消',
'voice.composer.insertText': '插入文字',
'voice.composer.discard': '丢弃',
'voice.composer.dismiss': '关闭提示',
'voice.composer.pendingTitle': '语音识别结果',
'voice.composer.pendingHint': '识别期间输入框内容发生了变化,未自动写入。',
'voice.composer.error.permission': '麦克风权限被拒绝,请在系统设置中允许后重试。',
'voice.composer.error.noDevice': '未检测到麦克风,请连接后重试。',
'voice.composer.error.deviceBusy': '麦克风正被其他应用占用。',
'voice.composer.error.unavailable': '当前环境不支持语音输入。',
'voice.composer.error.interrupted': '麦克风意外中断,本次录音已丢弃。',
'voice.composer.error.notReady': '语音模型尚未就绪,请在「设置 → 语音输入」中检查。',
'voice.composer.error.invalidAudio': '无法读取录音,请重试。',
'voice.composer.error.unknownProvider': '所选的语音引擎不可用。',
'voice.composer.error.failed': '识别失败,请重试。',
'voice.composer.error.noSpeech': '未识别到语音。',
'voice.composer.error.tooShort': '录音时间太短。',
}
+141 -1
View File
@@ -29,6 +29,15 @@ const mocks = vi.hoisted(() => ({
webviewUnlisten: vi.fn(),
isMobile: false,
isTauriRuntime: false,
voiceSupported: vi.fn(() => false),
voiceStartRecording: vi.fn(),
voiceTranscribe: vi.fn(),
voiceCatalog: vi.fn(async () => ({
supported: false,
providers: [],
preferences: { enabled: false, providerId: 'sensevoice-local', language: 'auto' },
limits: { maxAudioSeconds: 60, maxAudioBytes: 1_000_000 },
})),
}))
vi.mock('../api/sessions', () => ({
@@ -41,6 +50,24 @@ vi.mock('../api/sessions', () => ({
},
}))
vi.mock('@/api/voice', () => ({
voiceApi: {
catalog: mocks.voiceCatalog,
transcribe: mocks.voiceTranscribe,
providerStatus: vi.fn(),
updatePreferences: vi.fn(),
prepare: vi.fn(),
cancelPrepare: vi.fn(),
removeAssets: vi.fn(),
},
}))
vi.mock('@/features/voiceInput/recorder', async (importOriginal) => ({
...(await importOriginal<typeof import('@/features/voiceInput/recorder')>()),
isVoiceCaptureSupported: mocks.voiceSupported,
startRecording: mocks.voiceStartRecording,
}))
vi.mock('../api/skills', () => ({
skillsApi: {
list: mocks.listSkills,
@@ -172,7 +199,8 @@ import { usePluginStore } from '../stores/pluginStore'
import { useWorkflowStore } from '../stores/workflowStore'
import type { RepositoryContextResult } from '../api/sessions'
import { browserHost } from '../lib/desktopHost/browserHost'
import { getComposerElement, getComposerText, setComposerText } from '../components/chat/composerTestUtils'
import { getComposerElement, getComposerText, getComposerView, setComposerSelection, setComposerText } from '../components/chat/composerTestUtils'
import { useVoiceInputStore } from '../stores/voiceInputStore'
function okRepositoryContext(overrides: Partial<RepositoryContextResult> = {}): RepositoryContextResult {
return {
@@ -260,6 +288,8 @@ describe('EmptySession', () => {
mocks.webviewDragHandlers.length = 0
mocks.isMobile = false
mocks.isTauriRuntime = false
mocks.voiceSupported.mockReturnValue(false)
useVoiceInputStore.setState({ catalog: null, loading: false, error: null })
useSettingsStore.setState({ locale: 'en', activeProviderName: null, permissionMode: 'default' })
useSessionStore.setState(initialSessionState, true)
useChatStore.setState(initialChatState, true)
@@ -1451,4 +1481,114 @@ describe('EmptySession', () => {
})
})
})
describe('voice input', () => {
let finishTranscription: (text: string) => void
let activeRecording: { getLevel: () => number; stop: ReturnType<typeof vi.fn>; cancel: ReturnType<typeof vi.fn> }
beforeEach(() => {
activeRecording = {
getLevel: () => 0,
stop: vi.fn(async () => ({ wav: new Blob(['wav']), seconds: 2 })),
cancel: vi.fn(),
}
mocks.voiceSupported.mockReturnValue(true)
useVoiceInputStore.setState({
catalog: {
supported: true,
providers: [{
info: { id: 'sensevoice-local', name: 'SenseVoice', location: 'local', languages: ['auto', 'zh'] },
preparation: { phase: 'ready' },
}],
preferences: { enabled: true, providerId: 'sensevoice-local', language: 'zh' },
limits: { maxAudioSeconds: 60, maxAudioBytes: 1_000_000 },
},
})
mocks.voiceStartRecording.mockImplementation(async () => activeRecording)
mocks.voiceTranscribe.mockImplementation(() => new Promise((resolve) => {
finishTranscription = (text) => resolve({ text, audioSeconds: 2, inferenceSeconds: 0.1 })
}))
})
async function dictate() {
await act(async () => {
fireEvent.click(screen.getByRole('button', { name: 'Dictate' }))
})
await act(async () => {
fireEvent.click(await screen.findByRole('button', { name: 'Stop recording and transcribe' }))
})
await screen.findByRole('button', { name: 'Transcribing…' })
}
it('puts the microphone between the model picker and the send button', () => {
render(<EmptySession />)
const model = screen.getByTestId('model-selector')
const voice = screen.getByTestId('voice-input')
const run = screen.getByRole('button', { name: 'Run' })
expect(model.nextElementSibling).toBe(voice)
expect(voice.nextElementSibling).toBe(run)
})
it('does not render the microphone until dictation is ready', () => {
mocks.voiceSupported.mockReturnValue(false)
render(<EmptySession />)
expect(screen.queryByTestId('voice-input')).toBeNull()
})
it('writes dictated text at the caret without starting a session', async () => {
render(<EmptySession />)
setComposerText('ab', 1)
vi.spyOn(getComposerView(), 'hasFocus').mockReturnValue(true)
setComposerSelection(1)
await dictate()
await act(async () => {
finishTranscription('你好')
})
expect(getComposerText()).toBe('a你好b')
expect(mocks.createSession).not.toHaveBeenCalled()
})
it('keeps the text aside when the draft was edited while it was being recognised', async () => {
render(<EmptySession />)
setComposerText('hello', 5)
await dictate()
setComposerText('hello there', 11)
await act(async () => {
finishTranscription('dictated')
})
expect(getComposerText()).toBe('hello there')
expect(screen.getByTestId('voice-input-pending-text')).toHaveTextContent('dictated')
})
it('holds a result that arrives during an IME composition in the composer', async () => {
render(<EmptySession />)
await dictate()
fireEvent.compositionStart(getComposerElement())
await act(async () => {
finishTranscription('你好')
})
expect(getComposerText()).toBe('')
expect(screen.getByTestId('voice-input-pending-text')).toHaveTextContent('你好')
})
it('releases the microphone when the page is left mid-recording', async () => {
const { unmount } = render(<EmptySession />)
await act(async () => {
fireEvent.click(screen.getByRole('button', { name: 'Dictate' }))
})
await screen.findByRole('button', { name: 'Stop recording and transcribe' })
unmount()
expect(activeRecording.cancel).toHaveBeenCalledTimes(1)
})
})
})
+11
View File
@@ -72,6 +72,8 @@ import { useCapabilityMenu } from '@/components/chat/useCapabilityMenu'
import type { AttachmentRef } from '../types/chat'
import type { PermissionMode } from '../types/settings'
import type { SlashCommandOption } from '../components/chat/composerUtils'
import { useComposerDictation } from '@/features/voiceInput/useComposerDictation'
import { VoiceInputButton } from '@/features/voiceInput/VoiceInputButton'
type Attachment = ComposerAttachment
@@ -190,6 +192,12 @@ export function EmptySession() {
: undefined
const draftModelLabel = draftRuntimeSelection?.modelId ?? currentModel?.name ?? currentModel?.id
const isMobileComposer = useMobileViewport() && !isDesktopRuntime()
const dictation = useComposerDictation({
composerRef,
draft: input,
blocked: isSubmitting,
contextKey: 'empty-session',
})
useEffect(() => {
composerRef.current?.focus()
@@ -847,6 +855,8 @@ export function EmptySession() {
onChange={handleComposerChange}
onKeyDown={handleComposerKeyDown}
onPaste={handleComposerPaste}
onCompositionStart={dictation.compositionHandlers.onCompositionStart}
onCompositionEnd={dictation.compositionHandlers.onCompositionEnd}
placeholder={t('empty.placeholder')}
// `min-w-0`: see ChatInput — an unbreakable long run (URL,
// hash) otherwise grows this flex item past the panel.
@@ -939,6 +949,7 @@ export function EmptySession() {
compact={isMobileComposer}
/>
<ModelSelector ref={modelSelectorRef} runtimeKey={DRAFT_RUNTIME_SELECTION_KEY} disabled={isSubmitting} compact={isMobileComposer} />
<VoiceInputButton dictation={dictation} blocked={isSubmitting} mobile={isMobileComposer} />
{/* Kept identical to ChatInput's send button — same
component, shape, size and icon. See the note there for
why the label went away. */}
+3
View File
@@ -22,6 +22,7 @@ import { useUIStore } from '../stores/uiStore'
import { AgentManager } from '../components/settings/AgentManager'
import { H5AccessSettings } from './settings/H5AccessSettings'
import { GeneralSettings } from './settings/GeneralSettings'
import { VoiceInputSettings } from './settings/VoiceInputSettings'
import { AboutSettings } from './settings/AboutSettings'
import { ProviderSettings } from './settings/ProviderSettings'
@@ -68,6 +69,7 @@ export function DesktopSettings() {
<div className="flex-1 flex flex-col gap-0.5">
<TabButton icon="dns" label={t('settings.tab.providers')} active={activeTab === 'providers'} onClick={() => setActiveTab('providers')} />
<TabButton icon="tune" label={t('settings.tab.general')} active={activeTab === 'general'} onClick={() => setActiveTab('general')} />
<TabButton icon="mic" label={t('settings.tab.voice')} active={activeTab === 'voice'} onClick={() => setActiveTab('voice')} />
<TabButton icon="qr_code_2" label={t('settings.tab.h5Access')} active={activeTab === 'h5Access'} onClick={() => setActiveTab('h5Access')} />
<TabButton icon="chat" label={t('settings.tab.adapters')} active={activeTab === 'adapters'} onClick={() => setActiveTab('adapters')} />
<TabButton icon="terminal" label={t('settings.tab.terminal')} active={activeTab === 'terminal'} onClick={() => setActiveTab('terminal')} />
@@ -92,6 +94,7 @@ export function DesktopSettings() {
{activeTab === 'providers' && <ProviderSettings />}
{activeTab === 'activity' && <ActivitySettings />}
{activeTab === 'general' && <GeneralSettings />}
{activeTab === 'voice' && <VoiceInputSettings />}
{activeTab === 'h5Access' && <H5AccessSettings />}
{activeTab === 'adapters' && <AdapterSettings />}
{activeTab === 'terminal' && <TerminalSettings showPreferences />}
@@ -0,0 +1,92 @@
import { cleanup, fireEvent, render, screen, waitFor, within } from '@testing-library/react'
import '@testing-library/jest-dom'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
vi.mock('@/api/voice', () => ({
voiceApi: {
catalog: vi.fn(),
updatePreferences: vi.fn(),
prepare: vi.fn(),
cancelPrepare: vi.fn(),
providerStatus: vi.fn(),
removeAssets: vi.fn(),
transcribe: vi.fn(),
},
}))
vi.mock('@/features/voiceInput/devices', () => ({ listAudioInputs: vi.fn().mockResolvedValue([]) }))
vi.mock('@/features/voiceInput/recorder', async (importOriginal) => ({
...(await importOriginal<typeof import('@/features/voiceInput/recorder')>()),
isVoiceCaptureSupported: vi.fn(() => true),
}))
// The browser (H5) shell renders these two panels; their own tests cover them.
vi.mock('./settings/ProviderSettings', () => ({ ProviderSettings: () => <div>provider-panel</div> }))
vi.mock('./settings/H5GeneralSettings', () => ({ H5GeneralSettings: () => <div>general-panel</div> }))
import { voiceApi } from '@/api/voice'
import { DesktopSettings } from './Settings'
import { H5Settings } from './settings/H5Settings'
import { useSettingsStore } from '../stores/settingsStore'
import { useUIStore } from '../stores/uiStore'
import { useVoiceInputStore } from '../stores/voiceInputStore'
const catalog = {
supported: true,
providers: [{
info: { id: 'sensevoice-local', name: 'SenseVoice (local)', location: 'local' as const, languages: ['auto' as const], downloadBytes: 1 },
preparation: { phase: 'unprepared' as const },
}],
preferences: { enabled: false, providerId: 'sensevoice-local', language: 'auto' as const },
limits: { maxAudioSeconds: 60, maxAudioBytes: 1_000_000 },
}
beforeEach(() => {
Object.defineProperty(HTMLElement.prototype, 'scrollIntoView', { configurable: true, writable: true, value: vi.fn() })
localStorage.clear()
useSettingsStore.setState({ locale: 'en' })
useUIStore.setState({ activeSettingsTab: 'providers', pendingSettingsTab: null })
useVoiceInputStore.setState({ catalog: null, loading: false, error: null })
vi.mocked(voiceApi.catalog).mockResolvedValue(catalog)
})
afterEach(() => {
cleanup()
useUIStore.setState({ activeSettingsTab: 'providers', pendingSettingsTab: null })
})
describe('Voice Input settings tab wiring', () => {
it('adds a Voice Input entry to the desktop rail that opens the panel and remembers the choice', async () => {
render(<DesktopSettings />)
fireEvent.click(within(screen.getByTestId('settings-navigation')).getByRole('button', { name: 'Voice Input' }))
expect(await screen.findByRole('heading', { level: 2, name: 'Voice Input' })).toBeInTheDocument()
expect(useUIStore.getState().activeSettingsTab).toBe('voice')
expect(localStorage.getItem('cc-haha-active-settings-tab')).toBe('voice')
})
it('opens the panel for a pending request, so other screens can deep-link to it', async () => {
useUIStore.setState({ pendingSettingsTab: 'voice' })
render(<DesktopSettings />)
expect(await screen.findByRole('heading', { level: 2, name: 'Voice Input' })).toBeInTheDocument()
expect(useUIStore.getState().pendingSettingsTab).toBeNull()
expect(screen.getByRole('button', { name: 'Voice Input', current: 'page' })).toBeInTheDocument()
})
it('restores the tab from storage on the next launch', async () => {
localStorage.setItem('cc-haha-active-settings-tab', 'voice')
vi.resetModules()
const fresh = await import('../stores/uiStore')
expect(fresh.useUIStore.getState().activeSettingsTab).toBe('voice')
})
it('keeps voice out of the browser shell: a stored voice tab falls back to model settings', async () => {
useUIStore.setState({ activeSettingsTab: 'voice' })
render(<H5Settings />)
expect(await screen.findByText('provider-panel')).toBeInTheDocument()
expect(screen.queryByRole('heading', { name: 'Voice Input' })).not.toBeInTheDocument()
expect(within(screen.getByRole('navigation', { name: 'Settings' })).getAllByRole('button')).toHaveLength(2)
await waitFor(() => expect(useUIStore.getState().activeSettingsTab).toBe('providers'))
})
})
@@ -0,0 +1,912 @@
import { act, cleanup, fireEvent, render, screen, waitFor, within } from '@testing-library/react'
import '@testing-library/jest-dom'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { ApiError } from '@/api/client'
import type {
VoiceCatalog,
VoiceLanguage,
VoicePreparationState,
VoiceProviderStatus,
VoiceTranscript,
} from '@/api/voice'
vi.mock('@/api/voice', () => ({
voiceApi: {
catalog: vi.fn(),
updatePreferences: vi.fn(),
prepare: vi.fn(),
cancelPrepare: vi.fn(),
providerStatus: vi.fn(),
removeAssets: vi.fn(),
transcribe: vi.fn(),
},
}))
vi.mock('@/features/voiceInput/devices', () => ({
listAudioInputs: vi.fn(),
}))
vi.mock('@/features/voiceInput/recorder', async (importOriginal) => ({
...(await importOriginal<typeof import('@/features/voiceInput/recorder')>()),
isVoiceCaptureSupported: vi.fn(),
startRecording: vi.fn(),
}))
import { voiceApi } from '@/api/voice'
import { listAudioInputs } from '@/features/voiceInput/devices'
import { getPreferredMicrophoneId } from '@/features/voiceInput/devicePreference'
import {
isVoiceCaptureSupported,
startRecording,
VoiceRecorderError,
type StartRecordingOptions,
} from '@/features/voiceInput/recorder'
import { useSettingsStore } from '@/stores/settingsStore'
import { useVoiceInputStore } from '@/stores/voiceInputStore'
import { VoiceInputSettings } from './VoiceInputSettings'
const api = vi.mocked(voiceApi)
const listInputs = vi.mocked(listAudioInputs)
const captureSupported = vi.mocked(isVoiceCaptureSupported)
const start = vi.mocked(startRecording)
const MODEL_BYTES = 241_357_257
function makeProvider(
preparation: VoicePreparationState,
overrides: Partial<VoiceProviderStatus['info']> = {},
): VoiceProviderStatus {
return {
info: {
id: 'sensevoice-local',
name: 'SenseVoice (local)',
location: 'local',
languages: ['auto', 'zh', 'en', 'ja', 'ko', 'yue'],
downloadBytes: MODEL_BYTES,
...overrides,
},
preparation,
}
}
function makeCatalog(
preparation: VoicePreparationState,
patch: Partial<VoiceCatalog> = {},
): VoiceCatalog {
return {
supported: true,
providers: [makeProvider(preparation)],
preferences: { enabled: false, providerId: 'sensevoice-local', language: 'auto' },
limits: { maxAudioSeconds: 60, maxAudioBytes: 5_000_000 },
...patch,
}
}
const READY: VoicePreparationState = { phase: 'ready' }
const UNPREPARED: VoicePreparationState = { phase: 'unprepared' }
async function renderPage(catalog: VoiceCatalog) {
api.catalog.mockResolvedValue(catalog)
render(<VoiceInputSettings />)
await screen.findByRole('heading', { name: 'Voice Input' })
await screen.findByTestId('voice-model-status')
}
/** The library Dropdown: a trigger button, then a listbox of role=option rows once opened. */
const trigger = (name: string) => screen.getByRole('button', { name })
function openPicker(name: string) {
fireEvent.click(trigger(name))
return screen.getByRole('listbox', { name })
}
const optionLabels = (list: HTMLElement) =>
within(list).getAllByRole('option').map(option => option.querySelector('.font-medium')?.textContent)
function fakeRecording(overrides: Partial<{ level: number; wav: Blob; seconds: number }> = {}) {
const wav = overrides.wav ?? new Blob(['wav'], { type: 'audio/wav' })
return {
getLevel: vi.fn(() => overrides.level ?? 0.5),
stop: vi.fn().mockResolvedValue({ wav, seconds: overrides.seconds ?? 5.6 }),
cancel: vi.fn(),
}
}
const TRANSCRIPT: VoiceTranscript = { text: 'hello from the microphone', audioSeconds: 5.6, inferenceSeconds: 0.104 }
let mediaDevices: EventTarget
let objectUrlCounter: number
const createObjectURL = vi.fn()
const revokeObjectURL = vi.fn()
beforeEach(() => {
vi.clearAllMocks()
localStorage.clear()
useSettingsStore.setState({ locale: 'en' })
useVoiceInputStore.setState({ catalog: null, loading: false, error: null })
captureSupported.mockReturnValue(true)
listInputs.mockResolvedValue([])
api.updatePreferences.mockImplementation(async (patch) => ({
preferences: { ...useVoiceInputStore.getState().catalog!.preferences, ...patch },
}))
mediaDevices = new EventTarget()
Object.defineProperty(navigator, 'mediaDevices', { value: mediaDevices, configurable: true })
objectUrlCounter = 0
createObjectURL.mockImplementation(() => `blob:voice-${++objectUrlCounter}`)
Object.defineProperty(URL, 'createObjectURL', { value: createObjectURL, configurable: true, writable: true })
Object.defineProperty(URL, 'revokeObjectURL', { value: revokeObjectURL, configurable: true, writable: true })
})
afterEach(async () => {
cleanup()
// Stop any status poll a test left running so it cannot leak into the next one.
api.cancelPrepare.mockResolvedValue(makeProvider(UNPREPARED))
await useVoiceInputStore.getState().cancelPrepare('sensevoice-local')
Reflect.deleteProperty(navigator, 'mediaDevices')
useSettingsStore.setState(useSettingsStore.getInitialState(), true)
})
describe('VoiceInputSettings loading states', () => {
it('shows a retryable error when the catalog cannot be loaded', async () => {
api.catalog.mockRejectedValueOnce(new Error('offline'))
render(<VoiceInputSettings />)
expect(await screen.findByText('Could not load voice input settings.')).toBeInTheDocument()
expect(screen.getByRole('alert')).toHaveTextContent('offline')
api.catalog.mockResolvedValue(makeCatalog(UNPREPARED))
fireEvent.click(screen.getByRole('button', { name: 'Retry' }))
expect(await screen.findByTestId('voice-model-status')).toBeInTheDocument()
})
it('replaces the whole page with a notice when the platform is unsupported', async () => {
api.catalog.mockResolvedValue(makeCatalog(UNPREPARED, { supported: false }))
render(<VoiceInputSettings />)
expect(await screen.findByText('Local speech recognition is not supported on this platform yet')).toBeInTheDocument()
expect(screen.queryByRole('switch')).not.toBeInTheDocument()
expect(screen.queryByTestId('voice-model-status')).not.toBeInTheDocument()
expect(screen.queryByRole('button', { name: 'Start test' })).not.toBeInTheDocument()
})
it('re-reads the catalog every time the tab is entered', async () => {
useVoiceInputStore.setState({ catalog: makeCatalog(UNPREPARED) })
api.catalog.mockResolvedValue(makeCatalog(READY))
render(<VoiceInputSettings />)
await waitFor(() => expect(screen.getByTestId('voice-model-status')).toHaveAttribute('data-phase', 'ready'))
expect(api.catalog).toHaveBeenCalledTimes(1)
})
})
describe('VoiceInputSettings layout', () => {
it('keeps three cards under short headings: recognition, microphone, test', async () => {
await renderPage(makeCatalog(READY))
expect(screen.getAllByRole('heading', { level: 2 }).map(heading => heading.textContent)).toEqual([
'Voice Input',
'Recognition engine',
'Microphone',
'Transcription test',
])
// Enable, engine, model and language are rows of the first card, not cards of their own.
const card = screen.getByRole('switch', { name: 'Enable voice input' }).closest('section')!
expect(within(card).getByRole('button', { name: 'Engine' })).toBeInTheDocument()
expect(within(card).getByTestId('voice-model-status')).toBeInTheDocument()
expect(within(card).getByRole('button', { name: 'Language' })).toBeInTheDocument()
expect(screen.getByRole('button', { name: 'Input device' }).closest('section')).not.toBe(card)
// No bordered box nested in the card.
expect(screen.getByTestId('voice-model-status').className).not.toMatch(/\bborder\b/)
})
it('separates the rows of a card with the divider token', async () => {
await renderPage(makeCatalog(READY))
const rows = screen.getByTestId('voice-model-status').parentElement!
expect(rows.className).toContain('divide-y')
expect(rows.className).toContain('--color-border-separator')
})
})
describe('VoiceInputSettings model status', () => {
it('shows a quiet local size line from the provider, and not for a cloud provider or an unknown size', async () => {
await renderPage(makeCatalog(UNPREPARED))
expect(screen.getByText('Local recognition · about 230 MB')).toBeInTheDocument()
cleanup()
useVoiceInputStore.setState({ catalog: null })
const cloud = makeCatalog(UNPREPARED)
cloud.providers[0]!.info.location = 'cloud'
await renderPage(cloud)
expect(screen.queryByText(/Local recognition/)).not.toBeInTheDocument()
cleanup()
useVoiceInputStore.setState({ catalog: null })
const unknown = makeCatalog(UNPREPARED)
delete unknown.providers[0]!.info.downloadBytes
await renderPage(unknown)
expect(screen.queryByText(/Local recognition/)).not.toBeInTheDocument()
})
it('offers the download and starts it on click', async () => {
await renderPage(makeCatalog(UNPREPARED))
expect(screen.getByText('The speech model is not downloaded.')).toBeInTheDocument()
api.prepare.mockResolvedValue(makeProvider({ phase: 'downloading', step: 'runtime', resource: 'runtime.tar.gz', completedBytes: 0, totalBytes: 1000 }))
api.providerStatus.mockResolvedValue(makeProvider({ phase: 'downloading', step: 'runtime', resource: 'runtime.tar.gz', completedBytes: 0, totalBytes: 1000 }))
fireEvent.click(screen.getByRole('button', { name: 'Download' }))
await waitFor(() => expect(api.prepare).toHaveBeenCalledWith('sensevoice-local'))
await waitFor(() => expect(screen.getByTestId('voice-model-status')).toHaveAttribute('data-phase', 'downloading'))
// Downloading only starts from the explicit click, never from the toggle or the page load.
expect(api.prepare).toHaveBeenCalledTimes(1)
})
it('shows step, resource, byte counts, progress, resume note and source while downloading', async () => {
api.providerStatus.mockResolvedValue(makeProvider(UNPREPARED))
await renderPage(makeCatalog({
phase: 'downloading',
step: 'model',
resource: 'model.int8.onnx',
completedBytes: 100 * 1024 * 1024,
totalBytes: 200 * 1024 * 1024,
resumedFromBytes: 60 * 1024 * 1024,
source: 'https://hf-mirror.com/model.int8.onnx',
}))
const card = screen.getByTestId('voice-model-status')
expect(within(card).getByText(/Downloading speech model/)).toBeInTheDocument()
expect(within(card).getByText('· model.int8.onnx')).toBeInTheDocument()
expect(within(card).getByText('100 MB / 200 MB')).toBeInTheDocument()
expect(within(card).getByRole('progressbar')).toHaveAttribute('aria-valuenow', '50')
expect(within(card).getByText('Resumed from 60 MB that was already downloaded')).toBeInTheDocument()
expect(within(card).getByText('Source: https://hf-mirror.com/model.int8.onnx')).toBeInTheDocument()
})
it('omits the resume note for a fresh download and uses an indeterminate bar without a total', async () => {
api.providerStatus.mockResolvedValue(makeProvider(UNPREPARED))
await renderPage(makeCatalog({ phase: 'downloading', step: 'runtime', resource: 'runtime.tar.gz' }))
const card = screen.getByTestId('voice-model-status')
expect(within(card).queryByText(/Resumed from/)).not.toBeInTheDocument()
expect(within(card).getByRole('progressbar')).not.toHaveAttribute('aria-valuenow')
})
it('cancels a running download', async () => {
api.providerStatus.mockResolvedValue(makeProvider({ phase: 'downloading', step: 'model', completedBytes: 1, totalBytes: 2 }))
await renderPage(makeCatalog({ phase: 'downloading', step: 'model', completedBytes: 1, totalBytes: 2 }))
api.cancelPrepare.mockResolvedValue(makeProvider({ phase: 'cancelled' }))
fireEvent.click(screen.getByRole('button', { name: 'Cancel' }))
await waitFor(() => expect(api.cancelPrepare).toHaveBeenCalledWith('sensevoice-local'))
await waitFor(() => expect(screen.getByTestId('voice-model-status')).toHaveAttribute('data-phase', 'cancelled'))
})
it('reports verification without offering cancel', async () => {
api.providerStatus.mockResolvedValue(makeProvider({ phase: 'verifying' }))
await renderPage(makeCatalog({ phase: 'verifying' }))
const card = screen.getByTestId('voice-model-status')
expect(within(card).getAllByText('Verifying downloaded files…').length).toBeGreaterThan(0)
expect(within(card).queryByRole('button')).not.toBeInTheDocument()
})
it('shows a paused download as resumable', async () => {
await renderPage(makeCatalog({ phase: 'cancelled' }))
expect(screen.getByText(/Paused\. Parts already downloaded are kept/)).toBeInTheDocument()
api.prepare.mockResolvedValue(makeProvider({ phase: 'ready' }))
fireEvent.click(screen.getByRole('button', { name: 'Continue download' }))
await waitFor(() => expect(api.prepare).toHaveBeenCalledWith('sensevoice-local'))
})
it('confirms before deleting a ready model, and keeps it when the dialog is dismissed', async () => {
await renderPage(makeCatalog(READY))
expect(screen.getByText('Local model is ready. It wakes up automatically when you record.')).toBeInTheDocument()
fireEvent.click(screen.getByRole('button', { name: 'Delete model' }))
expect(await screen.findByText('Delete the local speech model?')).toBeInTheDocument()
expect(api.removeAssets).not.toHaveBeenCalled()
fireEvent.click(within(screen.getByRole('dialog')).getByRole('button', { name: 'Cancel' }))
await waitFor(() => expect(screen.queryByText('Delete the local speech model?')).not.toBeInTheDocument())
expect(api.removeAssets).not.toHaveBeenCalled()
fireEvent.click(screen.getByRole('button', { name: 'Delete model' }))
api.removeAssets.mockResolvedValue(makeProvider(UNPREPARED))
fireEvent.click(within(await screen.findByRole('dialog')).getByRole('button', { name: 'Delete' }))
await waitFor(() => expect(api.removeAssets).toHaveBeenCalledWith('sensevoice-local'))
await waitFor(() => expect(screen.getByTestId('voice-model-status')).toHaveAttribute('data-phase', 'unprepared'))
expect(screen.queryByText('Delete the local speech model?')).not.toBeInTheDocument()
})
it.each([
['network', 'Download failed: network connection problem'],
['dns', 'Download failed: could not resolve the download address'],
['timeout', 'Download failed: the connection timed out'],
['certificate', 'Download failed: the secure connection could not be verified'],
['http', 'Download failed: the server returned an error'],
['integrity', 'Download failed: the file check did not match, so the file will be downloaded again'],
['storage', 'Download failed: could not write to disk (check free space and permissions)'],
['unknown', 'Download failed for an unknown reason'],
] as const)('explains a %s failure and retries the download', async (reason, text) => {
await renderPage(makeCatalog({
phase: 'failed',
error: { reason, source: 'https://huggingface.co/model.onnx', message: 'raw failure detail' },
}))
expect(screen.getByRole('alert', { name: '' })).toHaveTextContent(text)
expect(screen.getByText('raw failure detail')).toBeInTheDocument()
expect(screen.getByText('Source: https://huggingface.co/model.onnx')).toBeInTheDocument()
api.prepare.mockResolvedValue(makeProvider({ phase: 'ready' }))
fireEvent.click(screen.getByRole('button', { name: 'Retry' }))
await waitFor(() => expect(api.prepare).toHaveBeenCalledWith('sensevoice-local'))
})
it('tells network-class failures that the downloaded part is kept, but not integrity failures', async () => {
await renderPage(makeCatalog({ phase: 'failed', error: { reason: 'network', message: 'boom' } }))
expect(screen.getByText(/Parts already downloaded are kept\. Retry once the network is back/)).toBeInTheDocument()
cleanup()
useVoiceInputStore.setState({ catalog: null })
await renderPage(makeCatalog({ phase: 'failed', error: { reason: 'integrity', message: 'bad hash' } }))
expect(screen.queryByText(/Retry once the network is back/)).not.toBeInTheDocument()
})
it('does not offer a retry on a platform the runtime cannot support', async () => {
await renderPage(makeCatalog({ phase: 'failed', error: { reason: 'unsupported-platform', message: 'no runtime' } }))
expect(screen.getByText('This platform is not supported by the local speech runtime')).toBeInTheDocument()
expect(screen.queryByRole('button', { name: 'Retry' })).not.toBeInTheDocument()
})
})
describe('VoiceInputSettings partial downloads', () => {
it.each([
['cancelled', { phase: 'cancelled', completedBytes: 50 * 1024 * 1024, totalBytes: 200 * 1024 * 1024 }],
['failed', { phase: 'failed', completedBytes: 50 * 1024 * 1024, error: { reason: 'network', message: 'boom' } }],
['unprepared', { phase: 'unprepared', completedBytes: 50 * 1024 * 1024 }],
] as const)('lets a %s download with bytes on disk be deleted, after confirming', async (phase, preparation) => {
await renderPage(makeCatalog(preparation as VoicePreparationState))
expect(screen.getByTestId('voice-model-status')).toHaveAttribute('data-phase', phase)
fireEvent.click(screen.getByRole('button', { name: 'Delete model' }))
expect(await screen.findByText('Delete the local speech model?')).toBeInTheDocument()
expect(api.removeAssets).not.toHaveBeenCalled()
api.removeAssets.mockResolvedValue(makeProvider(UNPREPARED))
fireEvent.click(within(screen.getByRole('dialog')).getByRole('button', { name: 'Delete' }))
await waitFor(() => expect(api.removeAssets).toHaveBeenCalledWith('sensevoice-local'))
})
it.each([
['cancelled', { phase: 'cancelled' }],
['failed', { phase: 'failed', completedBytes: 0, error: { reason: 'network', message: 'boom' } }],
['unprepared', { phase: 'unprepared' }],
] as const)('offers no delete for a %s download with nothing on disk', async (_phase, preparation) => {
await renderPage(makeCatalog(preparation as VoicePreparationState))
expect(screen.queryByRole('button', { name: 'Delete model' })).not.toBeInTheDocument()
})
it('does not offer delete while a download is running', async () => {
api.providerStatus.mockResolvedValue(makeProvider({ phase: 'downloading', completedBytes: 5, totalBytes: 10 }))
await renderPage(makeCatalog({ phase: 'downloading', completedBytes: 5, totalBytes: 10 }))
expect(screen.queryByRole('button', { name: 'Delete model' })).not.toBeInTheDocument()
})
})
describe('VoiceInputSettings preferences', () => {
it('saves the enable switch and warns, without downloading, when the model is missing', async () => {
await renderPage(makeCatalog(UNPREPARED))
expect(screen.queryByText(/Voice input is on, but the speech model is not downloaded yet/)).not.toBeInTheDocument()
fireEvent.click(screen.getByRole('switch', { name: 'Enable voice input' }))
await waitFor(() => expect(api.updatePreferences).toHaveBeenCalledWith({ enabled: true }))
expect(await screen.findByText(/Voice input is on, but the speech model is not downloaded yet/)).toBeInTheDocument()
expect(api.prepare).not.toHaveBeenCalled()
})
it('does not warn about a missing model once it is ready', async () => {
await renderPage(makeCatalog(READY, { preferences: { enabled: true, providerId: 'sensevoice-local', language: 'auto' } }))
expect(screen.getByRole('switch', { name: 'Enable voice input' })).toBeChecked()
expect(screen.queryByText(/Voice input is on, but the speech model is not downloaded yet/)).not.toBeInTheDocument()
})
it('shows a save failure instead of pretending the switch worked', async () => {
await renderPage(makeCatalog(READY))
api.updatePreferences.mockRejectedValueOnce(new Error('disk full'))
fireEvent.click(screen.getByRole('switch', { name: 'Enable voice input' }))
expect(await screen.findByText('Could not save this setting. Please try again.')).toBeInTheDocument()
expect(screen.getByRole('switch', { name: 'Enable voice input' })).not.toBeChecked()
})
it('offers only the languages the provider lists, labelled, and saves the pick', async () => {
const catalog = makeCatalog(READY)
catalog.providers[0]!.info.languages = ['auto', 'zh', 'yue']
await renderPage(catalog)
expect(trigger('Language')).toHaveTextContent('Auto detect')
const list = openPicker('Language')
expect(optionLabels(list)).toEqual(['Auto detect', 'Chinese (Mandarin)', 'Cantonese'])
expect(within(list).getByRole('option', { name: 'Auto detect' })).toHaveAttribute('aria-selected', 'true')
fireEvent.click(within(list).getByRole('option', { name: 'Cantonese' }))
await waitFor(() => expect(api.updatePreferences).toHaveBeenCalledWith({ language: 'yue' }))
await waitFor(() => expect(trigger('Language')).toHaveTextContent('Cantonese'))
expect(screen.queryByRole('listbox')).not.toBeInTheDocument()
})
it('falls back to the raw code for a language it has no label for', async () => {
const catalog = makeCatalog(READY)
catalog.providers[0]!.info.languages = ['auto', 'fr' as VoiceLanguage]
await renderPage(catalog)
expect(within(openPicker('Language')).getByRole('option', { name: 'fr' })).toBeInTheDocument()
})
it('renders every provider and moves to a language the new provider supports', async () => {
const second: VoiceProviderStatus = makeProvider(READY, { id: 'other-engine', name: 'Other engine', languages: ['auto', 'en'] })
const catalog = makeCatalog(READY, {
preferences: { enabled: true, providerId: 'sensevoice-local', language: 'yue' },
})
catalog.providers.push(second)
await renderPage(catalog)
expect(optionLabels(openPicker('Engine'))).toEqual(['SenseVoice (local)', 'Other engine'])
fireEvent.click(screen.getByRole('option', { name: 'Other engine' }))
await waitFor(() => expect(api.updatePreferences).toHaveBeenCalledWith({ providerId: 'other-engine', language: 'auto' }))
})
it('renders a single provider as a normal engine control', async () => {
await renderPage(makeCatalog(READY))
expect(trigger('Engine')).toHaveTextContent('SenseVoice (local)')
expect(within(openPicker('Engine')).getAllByRole('option')).toHaveLength(1)
})
})
describe('VoiceInputSettings microphone', () => {
it('lists devices without asking for permission and stores the choice locally', async () => {
listInputs.mockResolvedValue([
{ deviceId: 'mic-a', label: 'Built-in Microphone' },
{ deviceId: 'mic-b', label: 'USB Microphone' },
])
await renderPage(makeCatalog(READY))
expect(listInputs).toHaveBeenCalledWith(undefined)
expect(trigger('Input device')).toHaveTextContent('System default')
const list = openPicker('Input device')
await within(list).findByRole('option', { name: 'USB Microphone' })
expect(optionLabels(list)).toEqual(['System default', 'Built-in Microphone', 'USB Microphone'])
// The system default is the empty id; it must show as the selected row.
expect(within(list).getByRole('option', { name: 'System default' })).toHaveAttribute('aria-selected', 'true')
expect(screen.queryByRole('button', { name: 'Allow microphone access to show device names' })).not.toBeInTheDocument()
fireEvent.click(within(list).getByRole('option', { name: 'USB Microphone' }))
expect(getPreferredMicrophoneId()).toBe('mic-b')
expect(trigger('Input device')).toHaveTextContent('USB Microphone')
expect(within(openPicker('Input device')).getByRole('option', { name: 'USB Microphone' })).toHaveAttribute('aria-selected', 'true')
fireEvent.click(screen.getByRole('option', { name: 'System default' }))
expect(getPreferredMicrophoneId()).toBeUndefined()
expect(trigger('Input device')).toHaveTextContent('System default')
expect(within(openPicker('Input device')).getByRole('option', { name: 'System default' })).toHaveAttribute('aria-selected', 'true')
expect(api.updatePreferences).not.toHaveBeenCalled()
})
it('unlocks device names on request and re-lists with the labels', async () => {
listInputs.mockResolvedValue([{ deviceId: 'mic-a', label: '' }])
await renderPage(makeCatalog(READY))
expect(await within(openPicker('Input device')).findByRole('option', { name: 'Microphone 1' })).toBeInTheDocument()
fireEvent.click(trigger('Input device'))
listInputs.mockResolvedValue([{ deviceId: 'mic-a', label: 'Built-in Microphone' }])
fireEvent.click(screen.getByRole('button', { name: 'Allow microphone access to show device names' }))
await waitFor(() => expect(listInputs).toHaveBeenLastCalledWith(expect.objectContaining({ requestPermission: true })))
expect(await within(openPicker('Input device')).findByRole('option', { name: 'Built-in Microphone' })).toBeInTheDocument()
expect(screen.queryByRole('button', { name: 'Allow microphone access to show device names' })).not.toBeInTheDocument()
})
const ALLOW = 'Allow microphone access to show device names'
it('reports a refused permission prompt when Chromium lists a single blank entry', async () => {
// Unauthorized or system-denied Chromium reports one audioinput with empty id and label.
listInputs.mockResolvedValue([{ deviceId: '', label: '' }])
await renderPage(makeCatalog(READY))
await screen.findByRole('button', { name: ALLOW })
expect(screen.queryByText(/Microphone access was denied/)).not.toBeInTheDocument()
fireEvent.click(screen.getByRole('button', { name: ALLOW }))
expect(await screen.findByText(/Microphone access was denied/)).toBeInTheDocument()
expect(screen.queryByText('No microphone was detected.')).not.toBeInTheDocument()
expect(screen.getByRole('button', { name: ALLOW })).toBeInTheDocument()
})
it('reports a refused permission prompt when ids come back without names', async () => {
listInputs.mockResolvedValue([{ deviceId: 'mic-a', label: '' }])
await renderPage(makeCatalog(READY))
await screen.findByRole('button', { name: ALLOW })
fireEvent.click(screen.getByRole('button', { name: ALLOW }))
expect(await screen.findByText(/Microphone access was denied/)).toBeInTheDocument()
})
it('reports a refusal the browser states explicitly, even with an empty device list', async () => {
listInputs.mockImplementation(async (options) => {
options?.onPermissionError?.(new DOMException('denied', 'NotAllowedError'))
return []
})
await renderPage(makeCatalog(READY))
fireEvent.click(await screen.findByRole('button', { name: ALLOW }))
expect(await screen.findByText(/Microphone access was denied/)).toBeInTheDocument()
expect(screen.queryByText('No microphone was detected.')).not.toBeInTheDocument()
})
it('does not call it a missing microphone before permission, when the browser lists one blank entry', async () => {
listInputs.mockResolvedValue([{ deviceId: '', label: '' }])
await renderPage(makeCatalog(READY))
expect(await screen.findByText(/The microphone opens briefly to unlock device names/)).toBeInTheDocument()
expect(screen.getByRole('button', { name: ALLOW })).toBeInTheDocument()
expect(screen.queryByText('No microphone was detected.')).not.toBeInTheDocument()
expect(screen.queryByRole('alert')).not.toBeInTheDocument()
})
it('says no microphone was found only when there really is no input device', async () => {
listInputs.mockResolvedValue([])
await renderPage(makeCatalog(READY))
expect(await screen.findByText('No microphone was detected.')).toBeInTheDocument()
fireEvent.click(screen.getByRole('button', { name: ALLOW }))
await waitFor(() => expect(listInputs).toHaveBeenLastCalledWith(expect.objectContaining({ requestPermission: true })))
expect(screen.getByText('No microphone was detected.')).toBeInTheDocument()
expect(screen.queryByText(/Microphone access was denied/)).not.toBeInTheDocument()
})
it('names a device that is in use when the permission stream cannot open', async () => {
listInputs.mockImplementation(async (options) => {
options?.onPermissionError?.(new DOMException('busy', 'NotReadableError'))
return [{ deviceId: 'mic-a', label: '' }]
})
await renderPage(makeCatalog(READY))
fireEvent.click(await screen.findByRole('button', { name: ALLOW }))
expect(await screen.findByText(/in use by another app/)).toBeInTheDocument()
expect(screen.queryByText(/Microphone access was denied/)).not.toBeInTheDocument()
})
it('clears the refusal once a later grant unlocks the names', async () => {
listInputs.mockResolvedValue([{ deviceId: '', label: '' }])
await renderPage(makeCatalog(READY))
fireEvent.click(await screen.findByRole('button', { name: ALLOW }))
await screen.findByText(/Microphone access was denied/)
listInputs.mockResolvedValue([{ deviceId: 'mic-a', label: 'Built-in Microphone' }])
act(() => { mediaDevices.dispatchEvent(new Event('devicechange')) })
expect(await within(openPicker('Input device')).findByRole('option', { name: 'Built-in Microphone' })).toBeInTheDocument()
expect(screen.queryByText(/Microphone access was denied/)).not.toBeInTheDocument()
})
it('refreshes the list when a device is plugged in', async () => {
listInputs.mockResolvedValue([{ deviceId: 'mic-a', label: 'Built-in Microphone' }])
await renderPage(makeCatalog(READY))
const list = openPicker('Input device')
await within(list).findByRole('option', { name: 'Built-in Microphone' })
listInputs.mockResolvedValue([
{ deviceId: 'mic-a', label: 'Built-in Microphone' },
{ deviceId: 'mic-c', label: 'Headset' },
])
act(() => { mediaDevices.dispatchEvent(new Event('devicechange')) })
expect(await within(list).findByRole('option', { name: 'Headset' })).toBeInTheDocument()
})
it('stops listening for device changes when the page closes', async () => {
listInputs.mockResolvedValue([{ deviceId: 'mic-a', label: 'Built-in Microphone' }])
await renderPage(makeCatalog(READY))
await waitFor(() => expect(listInputs).toHaveBeenCalledTimes(1))
cleanup()
mediaDevices.dispatchEvent(new Event('devicechange'))
await Promise.resolve()
expect(listInputs).toHaveBeenCalledTimes(1)
})
it('falls back to the system default, with a notice, when the saved device is gone', async () => {
localStorage.setItem('cc-haha-voice-input-device', 'mic-unplugged')
listInputs.mockResolvedValue([{ deviceId: 'mic-a', label: 'Built-in Microphone' }])
await renderPage(makeCatalog(READY))
expect(await screen.findByText('The microphone you chose earlier is not available. The system default is used instead.')).toBeInTheDocument()
expect(trigger('Input device')).toHaveTextContent('System default')
expect(within(openPicker('Input device')).getByRole('option', { name: 'System default' })).toHaveAttribute('aria-selected', 'true')
fireEvent.click(trigger('Input device'))
// The test records from the default device, not the vanished one.
const recording = fakeRecording()
start.mockResolvedValue(recording)
fireEvent.click(screen.getByRole('button', { name: 'Start test' }))
await waitFor(() => expect(start).toHaveBeenCalled())
expect(start.mock.calls[0]![0].deviceId).toBeUndefined()
})
it('keeps a saved device selected while names are still hidden', async () => {
localStorage.setItem('cc-haha-voice-input-device', 'mic-a')
listInputs.mockResolvedValue([])
await renderPage(makeCatalog(READY))
await waitFor(() => expect(trigger('Input device')).toHaveTextContent('Saved microphone'))
expect(screen.queryByText(/is not available\. The system default/)).not.toBeInTheDocument()
})
it('explains that recording needs a secure context instead of crashing', async () => {
captureSupported.mockReturnValue(false)
await renderPage(makeCatalog(READY))
expect(screen.getAllByText(/The microphone is not available in this context/)).toHaveLength(2)
expect(screen.queryByRole('button', { name: 'Input device' })).not.toBeInTheDocument()
expect(screen.queryByRole('button', { name: 'Start test' })).toBeDisabled()
expect(listInputs).not.toHaveBeenCalled()
})
})
describe('VoiceInputSettings transcription test', () => {
let frames: Array<FrameRequestCallback>
let now: number
const canvasContext = {
setTransform: vi.fn(), clearRect: vi.fn(), beginPath: vi.fn(), moveTo: vi.fn(), lineTo: vi.fn(), stroke: vi.fn(),
}
beforeEach(() => {
frames = []
now = 1_000
vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(canvasContext as never)
Object.defineProperty(HTMLCanvasElement.prototype, 'clientWidth', { configurable: true, get: () => 300 })
vi.stubGlobal('requestAnimationFrame', (callback: FrameRequestCallback) => {
frames.push(callback)
return frames.length
})
vi.stubGlobal('cancelAnimationFrame', vi.fn())
vi.spyOn(performance, 'now').mockImplementation(() => now)
})
afterEach(() => {
Reflect.deleteProperty(HTMLCanvasElement.prototype, 'clientWidth')
vi.unstubAllGlobals()
vi.restoreAllMocks()
})
/** Runs every frame queued so far (the wave and the clock each keep one). */
function runFrame() {
const batch = frames.splice(0)
if (batch.length === 0) throw new Error('no animation frame scheduled')
act(() => { for (const callback of batch) callback(now) })
}
it('is disabled with a hint until the model is downloaded', async () => {
await renderPage(makeCatalog(UNPREPARED))
expect(screen.getByRole('button', { name: 'Start test' })).toBeDisabled()
expect(screen.getByText('Download the speech model above before testing.')).toBeInTheDocument()
expect(start).not.toHaveBeenCalled()
})
it('records with the chosen device, draws the wave, and shows text, duration and timing', async () => {
localStorage.setItem('cc-haha-voice-input-device', 'mic-b')
listInputs.mockResolvedValue([
{ deviceId: 'mic-a', label: 'Built-in Microphone' },
{ deviceId: 'mic-b', label: 'USB Microphone' },
])
const catalog = makeCatalog(READY, { preferences: { enabled: true, providerId: 'sensevoice-local', language: 'zh' } })
await renderPage(catalog)
await waitFor(() => expect(trigger('Input device')).toHaveTextContent('USB Microphone'))
const recording = fakeRecording({ level: 0.42 })
start.mockResolvedValue(recording)
fireEvent.click(screen.getByRole('button', { name: 'Start test' }))
const stopButton = await screen.findByRole('button', { name: 'Stop' })
expect(start).toHaveBeenCalledTimes(1)
expect(start.mock.calls[0]![0]).toMatchObject({ deviceId: 'mic-b', maxSeconds: 30 })
// The wave is decorative (canvas, aria-hidden); the wrapper carries the name for screen readers.
const wave = screen.getByRole('img', { name: 'Microphone level' })
expect(wave.querySelector('canvas')).toHaveAttribute('aria-hidden', 'true')
runFrame()
expect(recording.getLevel).toHaveBeenCalled()
expect(canvasContext.stroke).toHaveBeenCalled()
expect(screen.getByTestId('voice-clock')).toHaveTextContent('0:00')
now += 3_200
runFrame()
expect(screen.getByTestId('voice-clock')).toHaveTextContent('0:03')
api.transcribe.mockResolvedValue(TRANSCRIPT)
fireEvent.click(stopButton)
expect(await screen.findByTestId('voice-transcript')).toHaveTextContent('hello from the microphone')
expect(screen.getByText('Audio 5.6 s · Inference 0.10 s')).toBeInTheDocument()
expect(recording.stop).toHaveBeenCalledTimes(1)
expect(api.transcribe).toHaveBeenCalledTimes(1)
const [wav, options] = api.transcribe.mock.calls[0]!
expect(wav).toBeInstanceOf(Blob)
expect(options).toMatchObject({ providerId: 'sensevoice-local', language: 'zh' })
const player = screen.getByLabelText('Play back the recording')
expect(player).toHaveAttribute('src', 'blob:voice-1')
expect(screen.getByRole('button', { name: 'Start test' })).toBeEnabled()
})
it('caps the test at the server limit when that is shorter than 30 seconds', async () => {
await renderPage(makeCatalog(READY, { limits: { maxAudioSeconds: 12, maxAudioBytes: 1_000_000 } }))
start.mockResolvedValue(fakeRecording())
fireEvent.click(screen.getByRole('button', { name: 'Start test' }))
await waitFor(() => expect(start).toHaveBeenCalled())
expect(start.mock.calls[0]![0].maxSeconds).toBe(12)
})
it('revokes the previous playback URL on a new test and the last one on unmount', async () => {
await renderPage(makeCatalog(READY))
api.transcribe.mockResolvedValue(TRANSCRIPT)
for (const expected of ['blob:voice-1', 'blob:voice-2']) {
start.mockResolvedValue(fakeRecording())
fireEvent.click(screen.getByRole('button', { name: 'Start test' }))
fireEvent.click(await screen.findByRole('button', { name: 'Stop' }))
await waitFor(() => expect(screen.getByLabelText('Play back the recording')).toHaveAttribute('src', expected))
}
expect(revokeObjectURL).toHaveBeenCalledTimes(1)
expect(revokeObjectURL).toHaveBeenCalledWith('blob:voice-1')
cleanup()
expect(revokeObjectURL).toHaveBeenCalledTimes(2)
expect(revokeObjectURL).toHaveBeenLastCalledWith('blob:voice-2')
})
it('keeps the recording playable when transcription fails', async () => {
await renderPage(makeCatalog(READY))
start.mockResolvedValue(fakeRecording())
api.transcribe.mockRejectedValue(new ApiError(500, { error: 'voice/failed', message: 'worker crashed' }))
fireEvent.click(screen.getByRole('button', { name: 'Start test' }))
fireEvent.click(await screen.findByRole('button', { name: 'Stop' }))
expect(await screen.findByText('Transcription failed. Try again.')).toBeInTheDocument()
expect(screen.getByLabelText('Play back the recording')).toBeInTheDocument()
expect(screen.queryByTestId('voice-transcript')).not.toBeInTheDocument()
})
it.each([
['voice/not-ready', 409, 'The speech model is not ready. Check Settings → Voice input.'],
['voice/invalid-audio', 400, 'The recording could not be read. Try again.'],
['voice/unknown-provider', 404, 'The selected speech engine is unavailable.'],
])('maps the server error %s', async (code, status, text) => {
await renderPage(makeCatalog(READY))
start.mockResolvedValue(fakeRecording())
api.transcribe.mockRejectedValue(new ApiError(status, { error: code, message: 'x' }))
fireEvent.click(screen.getByRole('button', { name: 'Start test' }))
fireEvent.click(await screen.findByRole('button', { name: 'Stop' }))
expect(await screen.findByText(text)).toBeInTheDocument()
})
it('falls back to a generic message when the server cannot be reached', async () => {
await renderPage(makeCatalog(READY))
start.mockResolvedValue(fakeRecording())
api.transcribe.mockRejectedValue(new TypeError('Failed to fetch'))
fireEvent.click(screen.getByRole('button', { name: 'Start test' }))
fireEvent.click(await screen.findByRole('button', { name: 'Stop' }))
expect(await screen.findByText('Transcription failed. Try again.')).toBeInTheDocument()
})
it('says when nothing was recognized', async () => {
await renderPage(makeCatalog(READY))
start.mockResolvedValue(fakeRecording())
api.transcribe.mockResolvedValue({ text: ' ', audioSeconds: 1, inferenceSeconds: 0.02 })
fireEvent.click(screen.getByRole('button', { name: 'Start test' }))
fireEvent.click(await screen.findByRole('button', { name: 'Stop' }))
expect(await screen.findByText('No speech was recognized.')).toBeInTheDocument()
})
it.each([
['unavailable', 'Dictation is not available in this environment.'],
['permission', 'Microphone access was denied. Allow it in your system settings and try again.'],
['no-device', 'No microphone was found. Connect one and try again.'],
['device-busy', 'The microphone is in use by another app.'],
['interrupted', 'The microphone stopped unexpectedly. Recording was discarded.'],
['failed', 'Transcription failed. Try again.'],
] as const)('shows a clear message when the recorder fails to start with %s', async (code, text) => {
await renderPage(makeCatalog(READY))
start.mockRejectedValue(new VoiceRecorderError(code))
fireEvent.click(screen.getByRole('button', { name: 'Start test' }))
expect(await screen.findByRole('alert')).toHaveTextContent(text)
expect(screen.getByRole('button', { name: 'Start test' })).toBeEnabled()
expect(api.transcribe).not.toHaveBeenCalled()
})
it('transcribes on its own when the recorder hits the length cap', async () => {
await renderPage(makeCatalog(READY))
const recording = fakeRecording()
let options: StartRecordingOptions | undefined
start.mockImplementation(async (received) => {
options = received
return recording
})
api.transcribe.mockResolvedValue(TRANSCRIPT)
fireEvent.click(screen.getByRole('button', { name: 'Start test' }))
await screen.findByRole('button', { name: 'Stop' })
act(() => { options!.onLimitReached!() })
expect(await screen.findByTestId('voice-transcript')).toBeInTheDocument()
expect(recording.stop).toHaveBeenCalledTimes(1)
})
it('stops with an error when the device disappears mid-recording', async () => {
await renderPage(makeCatalog(READY))
const recording = fakeRecording()
let options: StartRecordingOptions | undefined
start.mockImplementation(async (received) => {
options = received
return recording
})
fireEvent.click(screen.getByRole('button', { name: 'Start test' }))
await screen.findByRole('button', { name: 'Stop' })
act(() => { options!.onInterrupted!(new VoiceRecorderError('interrupted')) })
expect(await screen.findByRole('alert')).toHaveTextContent('The microphone stopped unexpectedly')
expect(recording.cancel).toHaveBeenCalled()
expect(api.transcribe).not.toHaveBeenCalled()
expect(screen.getByRole('button', { name: 'Start test' })).toBeEnabled()
})
it('releases the microphone and aborts the upload when the page is left mid-test', async () => {
await renderPage(makeCatalog(READY))
const recording = fakeRecording()
start.mockResolvedValue(recording)
fireEvent.click(screen.getByRole('button', { name: 'Start test' }))
await screen.findByRole('button', { name: 'Stop' })
cleanup()
expect(recording.cancel).toHaveBeenCalled()
expect(api.transcribe).not.toHaveBeenCalled()
})
it('cancels a recording that finishes opening after the page was left', async () => {
await renderPage(makeCatalog(READY))
const recording = fakeRecording()
let resolveStart!: (value: ReturnType<typeof fakeRecording>) => void
start.mockImplementation(() => new Promise(resolve => { resolveStart = resolve }))
fireEvent.click(screen.getByRole('button', { name: 'Start test' }))
await waitFor(() => expect(start).toHaveBeenCalled())
const signal = start.mock.calls[0]![0].signal
cleanup()
expect(signal?.aborted).toBe(true)
await act(async () => { resolveStart(recording) })
expect(recording.cancel).toHaveBeenCalled()
})
})
@@ -0,0 +1,517 @@
import { useEffect, useState, type ReactNode } from 'react'
import { isVoiceCaptureSupported } from '@/features/voiceInput/recorder'
import type {
VoiceFailureReason,
VoiceLanguage,
VoicePreferences,
VoicePreparationPhase,
VoicePreparationStep,
VoiceProviderStatus,
} from '@/api/voice'
import { Badge, type Tone } from '@/components/ui/Badge'
import { Button } from '@/components/ui/Button'
import { Card } from '@/components/ui/Card'
import { Dropdown } from '@/components/ui/Dropdown'
import { ConfirmDialog } from '@/components/ui/ConfirmDialog'
import { ErrorState } from '@/components/ui/ErrorState'
import { LoadingState } from '@/components/ui/LoadingState'
import { Progress } from '@/components/ui/Progress'
import { Switch } from '@/components/ui/Switch'
import { SettingsPageHeader, SettingsSection } from '@/components/settings/SettingsSection'
import { useTranslation } from '@/i18n'
import type { TranslationKey } from '@/i18n/locales/en'
import { formatBytes } from '@/lib/formatBytes'
import { selectActiveVoiceProvider, useVoiceInputStore } from '@/stores/voiceInputStore'
import { recorderErrorKey, useMicrophoneSelection } from './useMicrophoneSelection'
import { VoiceTranscriptionTest } from './VoiceTranscriptionTest'
/** Upper bound for one settings test; the server limit can only lower it. */
const TEST_MAX_SECONDS = 30
const PHASE_KEYS: Record<VoicePreparationPhase, TranslationKey> = {
unprepared: 'voice.settings.phase.unprepared',
downloading: 'voice.settings.phase.downloading',
verifying: 'voice.settings.phase.verifying',
ready: 'voice.settings.phase.ready',
failed: 'voice.settings.phase.failed',
cancelled: 'voice.settings.phase.cancelled',
}
const PHASE_TONES: Record<VoicePreparationPhase, Tone> = {
unprepared: 'neutral',
downloading: 'brand',
verifying: 'info',
ready: 'success',
failed: 'danger',
cancelled: 'warning',
}
const STEP_KEYS: Record<VoicePreparationStep, TranslationKey> = {
runtime: 'voice.settings.step.runtime',
model: 'voice.settings.step.model',
vad: 'voice.settings.step.vad',
verify: 'voice.settings.step.verify',
}
const FAILURE_KEYS: Record<VoiceFailureReason, TranslationKey> = {
network: 'voice.settings.failure.network',
dns: 'voice.settings.failure.dns',
timeout: 'voice.settings.failure.timeout',
certificate: 'voice.settings.failure.certificate',
http: 'voice.settings.failure.http',
integrity: 'voice.settings.failure.integrity',
storage: 'voice.settings.failure.storage',
'unsupported-platform': 'voice.settings.failure.unsupportedPlatform',
unknown: 'voice.settings.failure.unknown',
}
/** Failures that a later retry can plausibly fix without touching the machine. */
const RESUMABLE_FAILURES = new Set<VoiceFailureReason>(['network', 'dns', 'timeout', 'certificate', 'http'])
const LANGUAGE_KEYS: Record<VoiceLanguage, TranslationKey> = {
auto: 'voice.settings.language.auto',
zh: 'voice.settings.language.zh',
en: 'voice.settings.language.en',
ja: 'voice.settings.language.ja',
ko: 'voice.settings.language.ko',
yue: 'voice.settings.language.yue',
}
export function VoiceInputSettings() {
const t = useTranslation()
const catalog = useVoiceInputStore(state => state.catalog)
const storeError = useVoiceInputStore(state => state.error)
const loadCatalog = useVoiceInputStore(state => state.loadCatalog)
const updatePreferences = useVoiceInputStore(state => state.updatePreferences)
const prepare = useVoiceInputStore(state => state.prepare)
const cancelPrepare = useVoiceInputStore(state => state.cancelPrepare)
const removeAssets = useVoiceInputStore(state => state.removeAssets)
const [captureSupported] = useState(() => isVoiceCaptureSupported())
const [saveFailed, setSaveFailed] = useState(false)
const [actionPending, setActionPending] = useState(false)
const [removeConfirmOpen, setRemoveConfirmOpen] = useState(false)
const microphone = useMicrophoneSelection(captureSupported)
useEffect(() => {
// Always re-read on entering the tab: models can be removed or downloaded
// by another window while this page was closed.
void loadCatalog({ force: true })
}, [loadCatalog])
const header = (
<SettingsPageHeader
title={t('voice.settings.title')}
description={t('voice.settings.description')}
/>
)
if (!catalog) {
return (
<div className="max-w-2xl">
{header}
{storeError ? (
<ErrorState
size="lg"
title={t('voice.settings.loadFailed')}
detail={storeError}
retryLabel={t('common.retry')}
onRetry={() => { void loadCatalog({ force: true }) }}
/>
) : (
<LoadingState size="md" label={t('common.loading')} />
)}
</div>
)
}
if (!catalog.supported) {
return (
<div className="max-w-2xl">
{header}
<ErrorState
size="lg"
tone="strong"
title={t('voice.settings.unsupported.title')}
detail={t('voice.settings.unsupported.detail')}
/>
</div>
)
}
const { preferences, providers, limits } = catalog
const provider = selectActiveVoiceProvider({ catalog }) ?? providers[0]
const phase = provider?.preparation.phase
const modelReady = phase === 'ready'
const languages = provider ? withCurrent(provider.info.languages, preferences.language) : []
const savePreferences = async (patch: Partial<VoicePreferences>) => {
setSaveFailed(false)
try {
await updatePreferences(patch)
} catch {
setSaveFailed(true)
}
}
const changeProvider = (providerId: string) => {
const next = providers.find(item => item.info.id === providerId)
if (!next) return
const patch: Partial<VoicePreferences> = { providerId }
if (!next.info.languages.includes(preferences.language)) {
patch.language = next.info.languages.includes('auto') ? 'auto' : next.info.languages[0]
}
void savePreferences(patch)
}
const runAction = async (action: () => Promise<void>) => {
setActionPending(true)
try {
await action()
} finally {
setActionPending(false)
}
}
return (
<div className="max-w-2xl">
{header}
<SettingsSection title={t('voice.settings.engine.title')} description={t('voice.settings.engine.description')}>
<Card radius="xl" surface="low" padding="none" className={CARD_ROWS}>
<div className="space-y-2 px-4 py-3">
<Switch
checked={preferences.enabled}
onChange={(enabled) => { void savePreferences({ enabled }) }}
label={t('voice.settings.enable.label')}
description={t('voice.settings.enable.description')}
/>
{preferences.enabled && !modelReady && phase !== 'downloading' && phase !== 'verifying' ? (
<p role="status" className="text-[13px] leading-5 text-[var(--color-warning)]">
{t('voice.settings.enable.needModel')}
</p>
) : null}
{saveFailed ? (
<p role="alert" className="text-[13px] leading-5 text-[var(--color-error)]">
{t('voice.settings.saveFailed')}
</p>
) : null}
</div>
{provider ? (
<>
<SettingRow label={t('voice.settings.engine.provider')}>
<Picker
label={t('voice.settings.engine.provider')}
value={provider.info.id}
onChange={changeProvider}
items={providers.map(item => ({ value: item.info.id, label: item.info.name }))}
/>
</SettingRow>
<ModelStatus
provider={provider}
pending={actionPending}
onDownload={() => runAction(() => prepare(provider.info.id))}
onCancel={() => runAction(() => cancelPrepare(provider.info.id))}
onRemove={() => setRemoveConfirmOpen(true)}
/>
{storeError ? (
<div className="px-4 py-3">
<ErrorState size="sm" title={t('voice.settings.actionFailed')} detail={storeError} />
</div>
) : null}
<SettingRow label={t('voice.settings.language.label')}>
<Picker
label={t('voice.settings.language.label')}
value={preferences.language}
onChange={(language) => { void savePreferences({ language }) }}
items={languages.map(code => ({ value: code, label: languageLabel(t, code) }))}
/>
</SettingRow>
</>
) : null}
</Card>
</SettingsSection>
<SettingsSection title={t('voice.settings.mic.title')} description={t('voice.settings.mic.description')}>
<Card radius="xl" surface="low" padding="none" className={CARD_ROWS}>
{!captureSupported ? (
<p className="px-4 py-3 text-[13px] leading-5 text-[var(--color-text-tertiary)]">{t('voice.settings.capture.unsupported')}</p>
) : (
<>
<SettingRow label={t('voice.settings.mic.label')}>
<Picker
label={t('voice.settings.mic.label')}
value={microphone.selectedId}
onChange={microphone.select}
items={[
{ value: '', label: t('voice.settings.mic.systemDefault') },
// A saved device we cannot verify yet (names and ids stay
// hidden until permission) still has to be a real option, or
// the picker would claim "system default" while a device is set.
...(microphone.selectedId && !microphone.devices.some(device => device.deviceId === microphone.selectedId)
? [{ value: microphone.selectedId, label: t('voice.settings.mic.savedDevice') }]
: []),
...microphone.devices.map((device, index) => ({
value: device.deviceId,
label: device.label || t('voice.settings.mic.unnamed', { index: index + 1 }),
})),
]}
/>
</SettingRow>
{microphone.savedMissing || microphone.error || microphone.noInputDevices || microphone.needsPermission ? (
<div className="flex flex-col gap-2 px-4 py-3 sm:flex-row sm:items-center sm:justify-between sm:gap-6">
<div className="min-w-0 space-y-1 text-[13px] leading-5">
{microphone.savedMissing ? (
<p role="status" className="text-[var(--color-warning)]">{t('voice.settings.mic.missing')}</p>
) : null}
{microphone.error ? (
<p role="alert" className="text-[var(--color-error)]">{t(recorderErrorKey({ code: microphone.error }))}</p>
) : microphone.noInputDevices ? (
<p className="text-[var(--color-text-tertiary)]">{t('voice.settings.mic.noDevices')}</p>
) : microphone.needsPermission ? (
// Before permission the browser hides names (and often the
// devices themselves), so this is not a problem to report.
<p className="text-[var(--color-text-tertiary)]">{t('voice.settings.mic.allowHint')}</p>
) : null}
</div>
{microphone.needsPermission ? (
<Button
variant="secondary"
size="base"
className="shrink-0 self-start sm:self-auto"
loading={microphone.requesting}
onClick={() => { void microphone.requestPermission() }}
>
{t('voice.settings.mic.allowAccess')}
</Button>
) : null}
</div>
) : null}
</>
)}
</Card>
</SettingsSection>
<SettingsSection title={t('voice.settings.test.title')} description={t('voice.settings.test.description')}>
<Card radius="xl" surface="low" padding="none" className="p-4">
<VoiceTranscriptionTest
deviceId={microphone.selectedId || undefined}
providerId={provider?.info.id ?? preferences.providerId}
language={preferences.language}
maxSeconds={Math.max(1, Math.min(TEST_MAX_SECONDS, limits.maxAudioSeconds))}
ready={modelReady}
captureSupported={captureSupported}
/>
</Card>
</SettingsSection>
<ConfirmDialog
open={removeConfirmOpen && !!provider}
onClose={() => setRemoveConfirmOpen(false)}
loading={actionPending}
onConfirm={async () => {
if (provider) await runAction(() => removeAssets(provider.info.id))
setRemoveConfirmOpen(false)
}}
title={t('voice.settings.model.removeTitle')}
body={t('voice.settings.model.removeBody')}
confirmLabel={t('common.delete')}
cancelLabel={t('common.cancel')}
/>
</div>
)
}
type PickerItem<T extends string> = { value: T; label: string }
const CARD_ROWS = 'divide-y divide-[var(--color-border-separator)]'
/** One "name on the left, control on the right" line; stacks on narrow widths. */
function SettingRow({ label, children }: { label: string; children: ReactNode }) {
return (
<div className="flex flex-col gap-2 px-4 py-3 sm:flex-row sm:items-center sm:justify-between sm:gap-6">
<span className="min-w-0 text-sm font-medium text-[var(--color-text-primary)]">{label}</span>
<div className="w-full sm:w-64 sm:shrink-0">{children}</div>
</div>
)
}
/** The library Dropdown (not the native select, whose menu is the OS's own); `label` is its accessible name. */
function Picker<T extends string>({ label, value, items, onChange }: {
label: string
value: T
items: PickerItem<T>[]
onChange: (value: T) => void
}) {
const selected = items.find(item => item.value === value)
return (
<Dropdown<T>
items={items}
value={value}
onChange={onChange}
label={label}
width="100%"
maxHeight={320}
className="block w-full"
trigger={
<Button variant="secondary" size="md" block className="h-10 gap-3" aria-label={label}>
<span className="min-w-0 flex-1 truncate text-left">{selected?.label ?? ''}</span>
<span className="material-symbols-outlined flex-shrink-0 text-[18px] text-[var(--color-text-secondary)]">expand_more</span>
</Button>
}
/>
)
}
/** Keeps a saved language selectable even if the provider stopped advertising it. */
function withCurrent(languages: VoiceLanguage[], current: VoiceLanguage): VoiceLanguage[] {
return languages.includes(current) ? languages : [...languages, current]
}
function languageLabel(t: (key: TranslationKey) => string, code: string): string {
const key = (LANGUAGE_KEYS as Record<string, TranslationKey | undefined>)[code]
return key ? t(key) : code
}
type ModelStatusProps = {
provider: VoiceProviderStatus
pending: boolean
onDownload: () => void
onCancel: () => void
onRemove: () => void
}
function ModelStatus({ provider, pending, onDownload, onCancel, onRemove }: ModelStatusProps) {
const t = useTranslation()
const { info, preparation } = provider
const { phase } = preparation
const total = preparation.totalBytes ?? 0
const completed = preparation.completedBytes ?? 0
const percent = total > 0 ? (completed / total) * 100 : undefined
const error = preparation.error
// Abandoned or failed downloads leave a partial file (up to the full model size) on disk.
const hasPartial = (phase === 'unprepared' || phase === 'cancelled' || phase === 'failed') && completed > 0
const canRetry = phase === 'failed' && error?.reason !== 'unsupported-platform'
// One row: what it is on the left, what can be done about it on the right.
const removeButton = (
<Button variant="danger-ghost" size="base" onClick={onRemove}>
{t('voice.settings.model.remove')}
</Button>
)
return (
<div
data-testid="voice-model-status"
data-phase={phase}
className="space-y-2 px-4 py-3"
>
<div className="flex flex-wrap items-center justify-between gap-x-3 gap-y-2">
<div className="flex min-w-0 items-center gap-2">
<span className="min-w-0 truncate text-sm font-medium text-[var(--color-text-primary)]">{info.name}</span>
<Badge tone={PHASE_TONES[phase]} size="sm">{t(PHASE_KEYS[phase])}</Badge>
</div>
<div className="flex items-center gap-2">
{phase === 'unprepared' ? (
<Button variant="primary" size="base" loading={pending} onClick={onDownload}>
{t('voice.settings.model.download')}
</Button>
) : null}
{phase === 'downloading' ? (
<Button variant="secondary" size="base" loading={pending} onClick={onCancel}>
{t('common.cancel')}
</Button>
) : null}
{canRetry ? (
<Button variant="primary" size="base" loading={pending} onClick={onDownload}>
{t('common.retry')}
</Button>
) : null}
{phase === 'cancelled' ? (
<Button variant="primary" size="base" loading={pending} onClick={onDownload}>
{t('voice.settings.model.resume')}
</Button>
) : null}
{phase === 'ready' || hasPartial ? removeButton : null}
</div>
</div>
{info.location === 'local' && info.downloadBytes ? (
<p className="text-xs text-[var(--color-text-tertiary)]">
{t('voice.settings.model.meta', { size: formatBytes(info.downloadBytes) })}
</p>
) : null}
{phase === 'unprepared' ? (
<p className="text-[13px] leading-5 text-[var(--color-text-secondary)]">{t('voice.settings.model.unprepared')}</p>
) : null}
{phase === 'downloading' ? (
<div className="space-y-2">
<div className="flex flex-wrap items-baseline justify-between gap-x-3 gap-y-1 text-[13px] text-[var(--color-text-secondary)]">
<span className="min-w-0 break-all">
{preparation.step ? t(STEP_KEYS[preparation.step]) : t('voice.settings.model.downloading')}
{preparation.resource ? <span className="text-[var(--color-text-tertiary)]">{` · ${preparation.resource}`}</span> : null}
</span>
{total > 0 ? (
<span className="shrink-0 tabular-nums">
{t('voice.settings.model.progress', { completed: formatBytes(completed), total: formatBytes(total) })}
</span>
) : null}
</div>
<Progress
label={t('voice.settings.model.downloading')}
value={percent}
indeterminate={percent === undefined}
size="md"
/>
{preparation.resumedFromBytes ? (
<p className="text-xs text-[var(--color-text-tertiary)]">
{t('voice.settings.model.resumed', { size: formatBytes(preparation.resumedFromBytes) })}
</p>
) : null}
{preparation.source ? (
<p className="break-all text-xs text-[var(--color-text-tertiary)]">
{t('voice.settings.model.source', { source: preparation.source })}
</p>
) : null}
</div>
) : null}
{phase === 'verifying' ? (
<div className="space-y-2">
<p className="text-[13px] leading-5 text-[var(--color-text-secondary)]">{t('voice.settings.model.verifying')}</p>
<Progress label={t('voice.settings.model.verifying')} indeterminate size="md" />
</div>
) : null}
{phase === 'ready' ? (
<p className="text-[13px] leading-5 text-[var(--color-text-secondary)]">{t('voice.settings.model.ready')}</p>
) : null}
{phase === 'failed' ? (
<div className="space-y-1">
<p role="alert" className="text-[13px] font-medium leading-5 text-[var(--color-error)]">
{t(FAILURE_KEYS[error?.reason ?? 'unknown'])}
</p>
{error?.message ? (
<p className="break-words text-xs leading-5 text-[var(--color-text-tertiary)]">{error.message}</p>
) : null}
{error?.source ? (
<p className="break-all text-xs text-[var(--color-text-tertiary)]">
{t('voice.settings.model.source', { source: error.source })}
</p>
) : null}
{error && RESUMABLE_FAILURES.has(error.reason) ? (
<p className="text-xs text-[var(--color-text-tertiary)]">{t('voice.settings.model.resumeHint')}</p>
) : null}
</div>
) : null}
{phase === 'cancelled' ? (
<p className="text-[13px] leading-5 text-[var(--color-text-secondary)]">{t('voice.settings.model.cancelled')}</p>
) : null}
</div>
)
}
@@ -0,0 +1,244 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { Mic, Square } from 'lucide-react'
import { ApiError } from '@/api/client'
import { voiceApi, type VoiceLanguage, type VoiceTranscript } from '@/api/voice'
import { Button } from '@/components/ui/Button'
import { Card } from '@/components/ui/Card'
import { useTranslation } from '@/i18n'
import type { TranslationKey } from '@/i18n/locales/en'
import { VoiceWave } from '@/features/voiceInput/VoiceWave'
import { startRecording, type ActiveRecording, type RecordingResult } from '@/features/voiceInput/recorder'
import { recorderErrorKey, voiceErrorKey } from './useMicrophoneSelection'
type Phase = 'idle' | 'starting' | 'recording' | 'transcribing'
function transcribeErrorKey(error: unknown): TranslationKey {
return voiceErrorKey(error instanceof ApiError ? (error.body as { error?: unknown } | null)?.error : undefined)
}
function formatClock(totalSeconds: number): string {
const minutes = Math.floor(totalSeconds / 60)
const seconds = totalSeconds % 60
return `${minutes}:${String(seconds).padStart(2, '0')}`
}
type Props = {
deviceId?: string
providerId: string
language: VoiceLanguage
maxSeconds: number
/** The selected provider's model is installed and can transcribe. */
ready: boolean
/** `isVoiceCaptureSupported()`: false on plain-HTTP H5 or without a recorder. */
captureSupported: boolean
}
/**
* Record a few seconds, run them through the real transcription route, and show
* what came back — the same path the composer uses, so a passing test means
* dictation will work. The wave and clock are drawn straight from requestAnimationFrame;
* routing 60 updates a second through React state would re-render the whole
* result card for a purely visual element.
*/
export function VoiceTranscriptionTest({ deviceId, providerId, language, maxSeconds, ready, captureSupported }: Props) {
const t = useTranslation()
const [phase, setPhase] = useState<Phase>('idle')
const [errorKey, setErrorKey] = useState<TranslationKey | null>(null)
const [transcript, setTranscript] = useState<VoiceTranscript | null>(null)
const [playbackUrl, setPlaybackUrl] = useState<string | null>(null)
const recordingRef = useRef<ActiveRecording | null>(null)
const startAbortRef = useRef<AbortController | null>(null)
const transcribeAbortRef = useRef<AbortController | null>(null)
const playbackUrlRef = useRef<string | null>(null)
const mountedRef = useRef(true)
const clockRef = useRef<HTMLSpanElement>(null)
// The limit/interrupt callbacks outlive the render that created them by up to
// `maxSeconds`, so they read the latest choices from here, not from closure.
const latestRef = useRef({ providerId, language })
latestRef.current = { providerId, language }
const replacePlayback = (blob: Blob | null) => {
if (playbackUrlRef.current) URL.revokeObjectURL(playbackUrlRef.current)
const next = blob ? URL.createObjectURL(blob) : null
playbackUrlRef.current = next
setPlaybackUrl(next)
}
useEffect(() => {
mountedRef.current = true
return () => {
mountedRef.current = false
startAbortRef.current?.abort()
transcribeAbortRef.current?.abort()
recordingRef.current?.cancel()
recordingRef.current = null
if (playbackUrlRef.current) URL.revokeObjectURL(playbackUrlRef.current)
playbackUrlRef.current = null
}
}, [])
useEffect(() => {
if (phase !== 'recording') return
const startedAt = performance.now()
let frame = 0
let shownSecond = -1
const tick = () => {
const elapsed = Math.floor((performance.now() - startedAt) / 1000)
if (elapsed !== shownSecond && clockRef.current) {
shownSecond = elapsed
clockRef.current.textContent = formatClock(elapsed)
}
frame = requestAnimationFrame(tick)
}
frame = requestAnimationFrame(tick)
return () => cancelAnimationFrame(frame)
}, [phase])
const readLevel = useCallback(() => recordingRef.current?.getLevel() ?? 0, [])
const finish = async () => {
const recording = recordingRef.current
if (!recording) return
recordingRef.current = null
setPhase('transcribing')
let recorded: RecordingResult
try {
recorded = await recording.stop()
} catch (error) {
if (!mountedRef.current) return
setErrorKey(recorderErrorKey(error))
setPhase('idle')
return
}
if (!mountedRef.current) return
replacePlayback(recorded.wav)
const controller = new AbortController()
transcribeAbortRef.current = controller
try {
const result = await voiceApi.transcribe(recorded.wav, { ...latestRef.current, signal: controller.signal })
if (mountedRef.current) setTranscript(result)
} catch (error) {
if (mountedRef.current && !controller.signal.aborted) setErrorKey(transcribeErrorKey(error))
} finally {
if (transcribeAbortRef.current === controller) transcribeAbortRef.current = null
if (mountedRef.current) setPhase('idle')
}
}
const handleInterrupted = () => {
const recording = recordingRef.current
recordingRef.current = null
recording?.cancel()
if (!mountedRef.current) return
setErrorKey('voice.composer.error.interrupted')
setPhase('idle')
}
const start = async () => {
setErrorKey(null)
setTranscript(null)
replacePlayback(null)
setPhase('starting')
const controller = new AbortController()
startAbortRef.current = controller
try {
const recording = await startRecording({
deviceId,
maxSeconds,
signal: controller.signal,
onLimitReached: () => { void finish() },
onInterrupted: handleInterrupted,
})
if (!mountedRef.current || controller.signal.aborted) {
recording.cancel()
return
}
recordingRef.current = recording
setPhase('recording')
} catch (error) {
if (!mountedRef.current) return
setErrorKey(recorderErrorKey(error))
setPhase('idle')
} finally {
if (startAbortRef.current === controller) startAbortRef.current = null
}
}
const canStart = ready && captureSupported
const busy = phase === 'starting' || phase === 'transcribing'
return (
<div className="space-y-3">
{!captureSupported ? (
<p className="text-[13px] leading-5 text-[var(--color-text-tertiary)]">{t('voice.settings.capture.unsupported')}</p>
) : !ready ? (
<p className="text-[13px] leading-5 text-[var(--color-text-tertiary)]">{t('voice.settings.test.needModel')}</p>
) : null}
<div className="flex flex-wrap items-center gap-3">
{phase === 'recording' ? (
<Button variant="danger" size="base" icon={<Square size={14} aria-hidden="true" />} onClick={() => void finish()}>
{t('voice.settings.test.stop')}
</Button>
) : (
<Button
variant="secondary"
size="base"
icon={<Mic size={14} aria-hidden="true" />}
loading={busy}
disabled={!canStart || busy}
onClick={() => void start()}
>
{phase === 'starting'
? t('voice.settings.test.starting')
: phase === 'transcribing'
? t('voice.settings.test.transcribing')
: t('voice.settings.test.start')}
</Button>
)}
{phase === 'recording' ? (
<div className="flex min-w-[180px] flex-1 items-center gap-3">
<div role="img" aria-label={t('voice.settings.test.level')} className="min-w-0 flex-1">
<VoiceWave getLevel={readLevel} active />
</div>
<span className="shrink-0 text-xs tabular-nums text-[var(--color-text-secondary)]">
<span ref={clockRef} data-testid="voice-clock">0:00</span>
{` / ${formatClock(maxSeconds)}`}
</span>
</div>
) : null}
</div>
{errorKey ? (
<p role="alert" className="text-[13px] leading-5 text-[var(--color-error)]">{t(errorKey)}</p>
) : null}
{transcript ? (
<Card radius="lg" surface="base" padding="md" className="space-y-2" aria-label={t('voice.settings.test.result')}>
<p className="text-xs font-medium text-[var(--color-text-secondary)]">{t('voice.settings.test.result')}</p>
{transcript.text.trim() ? (
<p data-testid="voice-transcript" className="whitespace-pre-wrap break-words text-sm leading-6 text-[var(--color-text-primary)]">
{transcript.text}
</p>
) : (
<p className="text-sm text-[var(--color-text-tertiary)]">{t('voice.composer.error.noSpeech')}</p>
)}
<p className="text-xs text-[var(--color-text-tertiary)]">
{t('voice.settings.test.stats', {
audio: transcript.audioSeconds.toFixed(1),
inference: transcript.inferenceSeconds.toFixed(2),
})}
</p>
</Card>
) : null}
{playbackUrl ? (
<audio controls src={playbackUrl} aria-label={t('voice.settings.test.playback')} className="h-9 w-full max-w-md" />
) : null}
</div>
)
}
@@ -0,0 +1,156 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { listAudioInputs, type AudioInputDevice } from '@/features/voiceInput/devices'
import { getPreferredMicrophoneId, setPreferredMicrophoneId } from '@/features/voiceInput/devicePreference'
import type { VoiceRecorderErrorCode } from '@/features/voiceInput/recorder'
import type { TranslationKey } from '@/i18n/locales/en'
/** The recorder error code carried by a thrown value, or `failed` for anything else. */
export function recorderErrorCode(error: unknown): VoiceRecorderErrorCode {
const code = (error as { code?: unknown } | null)?.code
switch (code) {
case 'unavailable':
case 'permission':
case 'no-device':
case 'device-busy':
case 'interrupted':
return code
default:
return 'failed'
}
}
// Recorder codes and server error codes share the composer's messages.
const ERROR_KEYS: Record<string, TranslationKey> = {
unavailable: 'voice.composer.error.unavailable',
permission: 'voice.composer.error.permission',
'no-device': 'voice.composer.error.noDevice',
'device-busy': 'voice.composer.error.deviceBusy',
interrupted: 'voice.composer.error.interrupted',
'voice/not-ready': 'voice.composer.error.notReady',
'voice/invalid-audio': 'voice.composer.error.invalidAudio',
'voice/unknown-provider': 'voice.composer.error.unknownProvider',
}
/** Anything without its own message (`failed`, `voice/failed`, unknown) reads as a failed transcription. */
export function voiceErrorKey(code: unknown): TranslationKey {
return (typeof code === 'string' && ERROR_KEYS[code]) || 'voice.composer.error.failed'
}
export function recorderErrorKey(error: unknown): TranslationKey {
return voiceErrorKey(recorderErrorCode(error))
}
/** What a failed `getUserMedia` says about the microphone, or null when it says nothing useful. */
function permissionFailureCode(error: unknown): VoiceRecorderErrorCode | null {
switch ((error as { name?: unknown } | null)?.name) {
case 'NotAllowedError':
case 'SecurityError':
case 'PermissionDeniedError':
return 'permission'
case 'NotFoundError':
case 'DevicesNotFoundError':
return 'no-device'
case 'NotReadableError':
case 'TrackStartError':
case 'AbortError':
return 'device-busy'
default:
return null
}
}
/**
* Microphone choice for the voice settings page.
*
* Listing never asks for permission on its own — that would pop the OS prompt
* just for opening a settings tab. Until the user grants it, browsers hide
* device names (and may hide ids), so a saved device is only reported as
* missing once real ids are visible and none of them matches.
*/
export function useMicrophoneSelection(enabled: boolean) {
const [devices, setDevices] = useState<AudioInputDevice[]>([])
const [loaded, setLoaded] = useState(false)
const [listedCount, setListedCount] = useState(0)
const [savedId, setSavedId] = useState<string | undefined>(() => getPreferredMicrophoneId())
const [error, setError] = useState<VoiceRecorderErrorCode | null>(null)
const [requesting, setRequesting] = useState(false)
// Why the last "allow access" attempt left the names hidden; cleared once names appear.
const [requestError, setRequestError] = useState<VoiceRecorderErrorCode | null>(null)
const requestSeqRef = useRef(0)
const mountedRef = useRef(true)
const refresh = useCallback(async (requestPermission = false) => {
const seq = ++requestSeqRef.current
if (requestPermission) setRequesting(true)
try {
let streamFailure: unknown
const list = await listAudioInputs(requestPermission
? { requestPermission: true, onPermissionError: (failure) => { streamFailure = failure } }
: undefined)
if (!mountedRef.current || seq !== requestSeqRef.current) return
// Unauthorized Chromium lists one blank entry (empty id and label), so the
// raw list is what tells "refused" apart from "no input device at all".
setDevices(list.filter(device => device.deviceId))
setListedCount(list.length)
if (list.some(device => device.label)) setRequestError(null)
else if (requestPermission) {
setRequestError(permissionFailureCode(streamFailure) ?? (list.length > 0 ? 'permission' : null))
}
setError(null)
} catch (caught) {
if (!mountedRef.current || seq !== requestSeqRef.current) return
setError(recorderErrorCode(caught))
} finally {
if (mountedRef.current && seq === requestSeqRef.current) {
setLoaded(true)
setRequesting(false)
}
}
}, [])
useEffect(() => {
mountedRef.current = true
return () => { mountedRef.current = false }
}, [])
useEffect(() => {
if (!enabled) return
void refresh()
const mediaDevices = typeof navigator === 'undefined' ? undefined : navigator.mediaDevices
if (!mediaDevices?.addEventListener) return
const onChange = () => { void refresh() }
mediaDevices.addEventListener('devicechange', onChange)
return () => mediaDevices.removeEventListener('devicechange', onChange)
}, [enabled, refresh])
const savedKnown = !!savedId && devices.some(device => device.deviceId === savedId)
const savedMissing = !!savedId && devices.length > 0 && !savedKnown
// '' is the system default; a saved id we cannot verify yet stays selected.
const selectedId = savedId && !savedMissing ? savedId : ''
const select = useCallback((id: string) => {
const next = id || undefined
setPreferredMicrophoneId(next)
setSavedId(next)
}, [])
const needsPermission = loaded && (devices.length === 0 || devices.some(device => !device.label))
// Only "the browser lists no input at all" is a missing microphone. Before
// permission Chromium lists one blank entry, which is not.
const noInputDevices = loaded && listedCount === 0
return {
devices,
loaded,
selectedId,
savedId,
savedMissing,
needsPermission,
noInputDevices,
requesting,
error: requestError ?? error,
select,
requestPermission: () => refresh(true),
}
}
+2
View File
@@ -47,6 +47,7 @@ const SETTINGS_TABS = [
'providers',
'activity',
'general',
'voice',
'h5Access',
'adapters',
'terminal',
@@ -234,6 +235,7 @@ export type SettingsTab =
| 'providers'
| 'activity'
| 'general'
| 'voice'
| 'h5Access'
| 'adapters'
| 'terminal'
+372
View File
@@ -0,0 +1,372 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { VoiceCatalog, VoicePreparationState, VoiceProviderStatus } from '@/api/voice'
vi.mock('@/api/voice', () => ({
voiceApi: {
catalog: vi.fn(),
updatePreferences: vi.fn(),
prepare: vi.fn(),
cancelPrepare: vi.fn(),
providerStatus: vi.fn(),
removeAssets: vi.fn(),
},
}))
import { voiceApi } from '@/api/voice'
import {
selectActiveVoiceProvider,
selectVoiceInputReady,
useVoiceInputStore,
} from './voiceInputStore'
const api = vi.mocked(voiceApi)
const ID = 'sensevoice-local'
function provider(preparation: VoicePreparationState, id = ID): VoiceProviderStatus {
return {
info: { id, name: id, location: 'local', languages: ['auto', 'zh'], downloadBytes: 1000 },
preparation,
}
}
function catalog(preparation: VoicePreparationState, patch: Partial<VoiceCatalog> = {}): VoiceCatalog {
return {
supported: true,
providers: [provider(preparation)],
preferences: { enabled: true, providerId: ID, language: 'auto' },
limits: { maxAudioSeconds: 60, maxAudioBytes: 1_000_000 },
...patch,
}
}
const phase = () => useVoiceInputStore.getState().catalog?.providers[0]!.preparation.phase
/** Lets pending promise continuations run without advancing the fake clock. */
const flush = () => vi.advanceTimersByTimeAsync(0)
beforeEach(() => {
vi.useFakeTimers()
vi.clearAllMocks()
useVoiceInputStore.setState({ catalog: null, loading: false, error: null })
})
afterEach(async () => {
// Poll timers live at module scope; cancelling is the public way to stop them.
api.cancelPrepare.mockResolvedValue(provider({ phase: 'unprepared' }))
await useVoiceInputStore.getState().cancelPrepare(ID)
vi.useRealTimers()
})
describe('voiceInputStore.loadCatalog', () => {
it('loads once and reuses the cached catalog', async () => {
api.catalog.mockResolvedValue(catalog({ phase: 'ready' }))
await useVoiceInputStore.getState().loadCatalog()
await useVoiceInputStore.getState().loadCatalog()
expect(api.catalog).toHaveBeenCalledTimes(1)
expect(useVoiceInputStore.getState().catalog?.supported).toBe(true)
})
it('reloads when forced', async () => {
api.catalog.mockResolvedValueOnce(catalog({ phase: 'unprepared' }))
await useVoiceInputStore.getState().loadCatalog()
api.catalog.mockResolvedValueOnce(catalog({ phase: 'ready' }))
await useVoiceInputStore.getState().loadCatalog({ force: true })
expect(api.catalog).toHaveBeenCalledTimes(2)
expect(phase()).toBe('ready')
})
it('joins an in-flight load instead of issuing a second request', async () => {
let resolve!: (value: VoiceCatalog) => void
api.catalog.mockImplementation(() => new Promise(r => { resolve = r }))
const first = useVoiceInputStore.getState().loadCatalog()
const second = useVoiceInputStore.getState().loadCatalog({ force: true })
resolve(catalog({ phase: 'ready' }))
await Promise.all([first, second])
expect(api.catalog).toHaveBeenCalledTimes(1)
})
it('records a failed load, clears the loading flag, and retries on the next call', async () => {
api.catalog.mockRejectedValueOnce(new Error('offline'))
await useVoiceInputStore.getState().loadCatalog()
expect(useVoiceInputStore.getState().error).toContain('offline')
expect(useVoiceInputStore.getState().loading).toBe(false)
expect(useVoiceInputStore.getState().catalog).toBeNull()
api.catalog.mockResolvedValueOnce(catalog({ phase: 'ready' }))
await useVoiceInputStore.getState().loadCatalog()
expect(useVoiceInputStore.getState().error).toBeNull()
expect(phase()).toBe('ready')
})
it('resumes polling when the server is already downloading, and stops at the terminal phase', async () => {
api.catalog.mockResolvedValue(catalog({ phase: 'downloading', completedBytes: 10, totalBytes: 100 }))
api.providerStatus
.mockResolvedValueOnce(provider({ phase: 'downloading', completedBytes: 60, totalBytes: 100 }))
.mockResolvedValueOnce(provider({ phase: 'ready' }))
await useVoiceInputStore.getState().loadCatalog()
expect(api.providerStatus).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(500)
expect(api.providerStatus).toHaveBeenCalledTimes(1)
expect(useVoiceInputStore.getState().catalog?.providers[0]!.preparation.completedBytes).toBe(60)
await vi.advanceTimersByTimeAsync(500)
expect(phase()).toBe('ready')
await vi.advanceTimersByTimeAsync(5000)
expect(api.providerStatus).toHaveBeenCalledTimes(2)
})
it('also resumes polling for a download in the verifying phase', async () => {
api.catalog.mockResolvedValue(catalog({ phase: 'verifying' }))
api.providerStatus.mockResolvedValue(provider({ phase: 'ready' }))
await useVoiceInputStore.getState().loadCatalog()
await vi.advanceTimersByTimeAsync(500)
expect(phase()).toBe('ready')
})
it.each(['ready', 'unprepared', 'failed', 'cancelled'] as const)('does not poll a catalog that is %s', async (value) => {
api.catalog.mockResolvedValue(catalog({ phase: value }))
await useVoiceInputStore.getState().loadCatalog()
await vi.advanceTimersByTimeAsync(3000)
expect(api.providerStatus).not.toHaveBeenCalled()
})
})
describe('voiceInputStore.prepare', () => {
beforeEach(async () => {
api.catalog.mockResolvedValue(catalog({ phase: 'unprepared' }))
await useVoiceInputStore.getState().loadCatalog()
})
it('polls until the download reaches a terminal phase and then stops', async () => {
api.prepare.mockResolvedValue(provider({ phase: 'downloading', step: 'runtime', completedBytes: 0, totalBytes: 100 }))
api.providerStatus
.mockResolvedValueOnce(provider({ phase: 'downloading', step: 'model', completedBytes: 50, totalBytes: 100 }))
.mockResolvedValueOnce(provider({ phase: 'verifying' }))
.mockResolvedValueOnce(provider({ phase: 'ready' }))
await useVoiceInputStore.getState().prepare(ID)
expect(api.prepare).toHaveBeenCalledWith(ID)
expect(phase()).toBe('downloading')
await vi.advanceTimersByTimeAsync(500)
expect(useVoiceInputStore.getState().catalog?.providers[0]!.preparation.step).toBe('model')
await vi.advanceTimersByTimeAsync(500)
expect(phase()).toBe('verifying')
await vi.advanceTimersByTimeAsync(500)
expect(phase()).toBe('ready')
await vi.advanceTimersByTimeAsync(10_000)
expect(api.providerStatus).toHaveBeenCalledTimes(3)
})
it.each([
['failed', { phase: 'failed', error: { reason: 'network', message: 'boom' } }],
['cancelled', { phase: 'cancelled' }],
] as const)('stops polling on a %s terminal phase', async (expected, terminal) => {
api.prepare.mockResolvedValue(provider({ phase: 'downloading', completedBytes: 0, totalBytes: 100 }))
api.providerStatus.mockResolvedValue(provider(terminal as VoicePreparationState))
await useVoiceInputStore.getState().prepare(ID)
await vi.advanceTimersByTimeAsync(500)
expect(phase()).toBe(expected)
await vi.advanceTimersByTimeAsync(10_000)
expect(api.providerStatus).toHaveBeenCalledTimes(1)
})
it('does not poll when the server answers with a terminal phase straight away', async () => {
api.prepare.mockResolvedValue(provider({ phase: 'ready' }))
await useVoiceInputStore.getState().prepare(ID)
await vi.advanceTimersByTimeAsync(2000)
expect(phase()).toBe('ready')
expect(api.providerStatus).not.toHaveBeenCalled()
})
it('keeps a single poll loop when prepare is joined twice', async () => {
api.prepare.mockResolvedValue(provider({ phase: 'downloading', completedBytes: 0, totalBytes: 100 }))
api.providerStatus.mockResolvedValue(provider({ phase: 'downloading', completedBytes: 1, totalBytes: 100 }))
await useVoiceInputStore.getState().prepare(ID)
await useVoiceInputStore.getState().prepare(ID)
await vi.advanceTimersByTimeAsync(500)
expect(api.providerStatus).toHaveBeenCalledTimes(1)
})
it('surfaces a failed request without touching the catalog, and clears the error on success', async () => {
api.prepare.mockRejectedValueOnce(new Error('server down'))
await useVoiceInputStore.getState().prepare(ID)
expect(useVoiceInputStore.getState().error).toContain('server down')
expect(phase()).toBe('unprepared')
api.prepare.mockResolvedValueOnce(provider({ phase: 'ready' }))
await useVoiceInputStore.getState().prepare(ID)
expect(useVoiceInputStore.getState().error).toBeNull()
})
it('gives up and reports the error after three failed status requests in a row', async () => {
api.prepare.mockResolvedValue(provider({ phase: 'downloading', completedBytes: 0, totalBytes: 100 }))
api.providerStatus.mockRejectedValue(new Error('lost server'))
await useVoiceInputStore.getState().prepare(ID)
await vi.advanceTimersByTimeAsync(500)
await vi.advanceTimersByTimeAsync(500)
expect(useVoiceInputStore.getState().error).toBeNull()
await vi.advanceTimersByTimeAsync(500)
expect(useVoiceInputStore.getState().error).toContain('lost server')
await vi.advanceTimersByTimeAsync(10_000)
expect(api.providerStatus).toHaveBeenCalledTimes(3)
})
it('rides out transient status failures and keeps the download progress moving', async () => {
api.prepare.mockResolvedValue(provider({ phase: 'downloading', completedBytes: 0, totalBytes: 100 }))
api.providerStatus
.mockRejectedValueOnce(new Error('blip'))
.mockRejectedValueOnce(new Error('blip'))
.mockResolvedValueOnce(provider({ phase: 'downloading', completedBytes: 40, totalBytes: 100 }))
.mockRejectedValueOnce(new Error('blip'))
.mockRejectedValueOnce(new Error('blip'))
.mockResolvedValueOnce(provider({ phase: 'ready' }))
await useVoiceInputStore.getState().prepare(ID)
await vi.advanceTimersByTimeAsync(500 * 6)
// Two failures, a success, two more failures: the streak was reset, so it never reached three.
expect(phase()).toBe('ready')
expect(useVoiceInputStore.getState().error).toBeNull()
})
})
describe('voiceInputStore.cancelPrepare and removeAssets', () => {
beforeEach(async () => {
api.catalog.mockResolvedValue(catalog({ phase: 'unprepared' }))
await useVoiceInputStore.getState().loadCatalog()
api.prepare.mockResolvedValue(provider({ phase: 'downloading', completedBytes: 0, totalBytes: 100 }))
api.providerStatus.mockResolvedValue(provider({ phase: 'downloading', completedBytes: 1, totalBytes: 100 }))
await useVoiceInputStore.getState().prepare(ID)
})
it('cancel stops the poll loop and applies the server answer', async () => {
api.cancelPrepare.mockResolvedValue(provider({ phase: 'cancelled' }))
await useVoiceInputStore.getState().cancelPrepare(ID)
await vi.advanceTimersByTimeAsync(10_000)
expect(api.cancelPrepare).toHaveBeenCalledWith(ID)
expect(phase()).toBe('cancelled')
expect(api.providerStatus).not.toHaveBeenCalled()
})
it('a late status answer cannot resurrect a cancelled download', async () => {
// The poll request is in flight when the user cancels; its response arrives after.
let resolveStatus!: (value: VoiceProviderStatus) => void
api.providerStatus.mockImplementation(() => new Promise(resolve => { resolveStatus = resolve }))
await vi.advanceTimersByTimeAsync(500)
api.cancelPrepare.mockResolvedValue(provider({ phase: 'cancelled' }))
await useVoiceInputStore.getState().cancelPrepare(ID)
resolveStatus(provider({ phase: 'downloading', completedBytes: 90, totalBytes: 100 }))
await flush()
await vi.advanceTimersByTimeAsync(10_000)
expect(api.providerStatus).toHaveBeenCalledTimes(1)
expect(phase()).toBe('cancelled')
})
it('removeAssets stops polling and applies the unprepared status', async () => {
api.removeAssets.mockResolvedValue(provider({ phase: 'unprepared' }))
await useVoiceInputStore.getState().removeAssets(ID)
await vi.advanceTimersByTimeAsync(10_000)
expect(api.removeAssets).toHaveBeenCalledWith(ID)
expect(phase()).toBe('unprepared')
expect(api.providerStatus).not.toHaveBeenCalled()
})
it('removeAssets reports a failure', async () => {
api.removeAssets.mockRejectedValue(new Error('locked'))
await useVoiceInputStore.getState().removeAssets(ID)
expect(useVoiceInputStore.getState().error).toContain('locked')
})
})
describe('voiceInputStore.updatePreferences', () => {
it('merges the server-confirmed preferences into the catalog, leaving providers alone', async () => {
api.catalog.mockResolvedValue(catalog({ phase: 'ready' }, { preferences: { enabled: false, providerId: ID, language: 'auto' } }))
await useVoiceInputStore.getState().loadCatalog()
api.updatePreferences.mockResolvedValue({ preferences: { enabled: true, providerId: ID, language: 'zh' } })
await useVoiceInputStore.getState().updatePreferences({ enabled: true, language: 'zh' })
expect(api.updatePreferences).toHaveBeenCalledWith({ enabled: true, language: 'zh' })
expect(useVoiceInputStore.getState().catalog?.preferences).toEqual({ enabled: true, providerId: ID, language: 'zh' })
expect(phase()).toBe('ready')
})
it('leaves the catalog unchanged and lets the caller see the failure', async () => {
api.catalog.mockResolvedValue(catalog({ phase: 'ready' }, { preferences: { enabled: false, providerId: ID, language: 'auto' } }))
await useVoiceInputStore.getState().loadCatalog()
api.updatePreferences.mockRejectedValue(new Error('read-only'))
await expect(useVoiceInputStore.getState().updatePreferences({ enabled: true })).rejects.toThrow('read-only')
expect(useVoiceInputStore.getState().catalog?.preferences.enabled).toBe(false)
})
})
describe('selectVoiceInputReady', () => {
const ready = catalog({ phase: 'ready' })
it('is true only when supported, enabled and the selected provider is ready', () => {
expect(selectVoiceInputReady({ catalog: ready })).toBe(true)
})
it('is false before the catalog loads', () => {
expect(selectVoiceInputReady({ catalog: null })).toBe(false)
})
it('is false on an unsupported platform even if everything else is set', () => {
expect(selectVoiceInputReady({ catalog: { ...ready, supported: false } })).toBe(false)
})
it('is false while disabled', () => {
expect(selectVoiceInputReady({
catalog: { ...ready, preferences: { ...ready.preferences, enabled: false } },
})).toBe(false)
})
it.each(['unprepared', 'downloading', 'verifying', 'failed', 'cancelled'] as const)('is false while the model is %s', (value) => {
expect(selectVoiceInputReady({ catalog: catalog({ phase: value }) })).toBe(false)
})
it('looks at the selected provider, not just any ready one', () => {
const other = provider({ phase: 'ready' }, 'other')
const state = {
catalog: catalog({ phase: 'unprepared' }, {
providers: [provider({ phase: 'unprepared' }), other],
}),
}
expect(selectVoiceInputReady(state)).toBe(false)
state.catalog.preferences.providerId = 'other'
expect(selectVoiceInputReady(state)).toBe(true)
})
it('is false when the selected provider is not in the catalog', () => {
expect(selectVoiceInputReady({
catalog: { ...ready, preferences: { ...ready.preferences, providerId: 'missing' } },
})).toBe(false)
expect(selectActiveVoiceProvider({
catalog: { ...ready, preferences: { ...ready.preferences, providerId: 'missing' } },
})).toBeUndefined()
})
})
+153
View File
@@ -0,0 +1,153 @@
import { create } from 'zustand'
import {
voiceApi,
type VoiceCatalog,
type VoicePreferences,
type VoiceProviderStatus,
} from '@/api/voice'
const POLL_INTERVAL_MS = 500
// A blip must not freeze the progress of a download the server is still running.
const MAX_CONSECUTIVE_POLL_FAILURES = 3
const TERMINAL_PHASES = new Set(['ready', 'failed', 'cancelled', 'unprepared'])
const pollTimers = new Map<string, ReturnType<typeof setTimeout>>()
// Bumped on every stop so a status request already in flight when the user
// cancels or deletes cannot land afterwards and restart the loop.
const pollGenerations = new Map<string, number>()
type VoiceInputState = {
catalog: VoiceCatalog | null
loading: boolean
error: string | null
/** Loads the catalog once; safe to call from every consumer. */
loadCatalog: (options?: { force?: boolean }) => Promise<void>
updatePreferences: (patch: Partial<VoicePreferences>) => Promise<void>
/** Starts (or joins) a download and polls until it reaches a terminal phase. */
prepare: (providerId: string) => Promise<void>
cancelPrepare: (providerId: string) => Promise<void>
removeAssets: (providerId: string) => Promise<void>
}
function replaceProvider(catalog: VoiceCatalog | null, status: VoiceProviderStatus): VoiceCatalog | null {
if (!catalog) return catalog
return {
...catalog,
providers: catalog.providers.map(item => item.info.id === status.info.id ? status : item),
}
}
export const useVoiceInputStore = create<VoiceInputState>((set, get) => {
const applyStatus = (status: VoiceProviderStatus) => {
set(state => ({ catalog: replaceProvider(state.catalog, status) }))
}
const stopPolling = (providerId: string) => {
const timer = pollTimers.get(providerId)
if (timer) clearTimeout(timer)
pollTimers.delete(providerId)
pollGenerations.set(providerId, (pollGenerations.get(providerId) ?? 0) + 1)
}
const poll = (providerId: string) => {
stopPolling(providerId)
const generation = pollGenerations.get(providerId)
let failures = 0
const tick = async () => {
try {
const status = await voiceApi.providerStatus(providerId)
if (pollGenerations.get(providerId) !== generation) return
failures = 0
applyStatus(status)
if (TERMINAL_PHASES.has(status.preparation.phase)) {
stopPolling(providerId)
return
}
} catch (error) {
if (pollGenerations.get(providerId) !== generation) return
failures += 1
if (failures >= MAX_CONSECUTIVE_POLL_FAILURES) {
set({ error: String(error) })
stopPolling(providerId)
return
}
}
pollTimers.set(providerId, setTimeout(tick, POLL_INTERVAL_MS))
}
pollTimers.set(providerId, setTimeout(tick, POLL_INTERVAL_MS))
}
return {
catalog: null,
loading: false,
error: null,
loadCatalog: async (options) => {
if (get().loading) return
if (get().catalog && !options?.force) return
set({ loading: true })
try {
const catalog = await voiceApi.catalog()
set({ catalog, error: null })
// Resume progress display if the server is already downloading.
for (const provider of catalog.providers) {
if (provider.preparation.phase === 'downloading' || provider.preparation.phase === 'verifying') {
poll(provider.info.id)
}
}
} catch (error) {
set({ error: String(error) })
} finally {
set({ loading: false })
}
},
updatePreferences: async (patch) => {
const { preferences } = await voiceApi.updatePreferences(patch)
set(state => ({ catalog: state.catalog ? { ...state.catalog, preferences } : state.catalog }))
},
prepare: async (providerId) => {
try {
const status = await voiceApi.prepare(providerId)
applyStatus(status)
set({ error: null })
if (!TERMINAL_PHASES.has(status.preparation.phase)) poll(providerId)
} catch (error) {
set({ error: String(error) })
}
},
cancelPrepare: async (providerId) => {
stopPolling(providerId)
try {
applyStatus(await voiceApi.cancelPrepare(providerId))
} catch (error) {
set({ error: String(error) })
}
},
removeAssets: async (providerId) => {
stopPolling(providerId)
try {
applyStatus(await voiceApi.removeAssets(providerId))
set({ error: null })
} catch (error) {
set({ error: String(error) })
}
},
}
})
/** The provider the user selected, or undefined before the catalog loads. */
export function selectActiveVoiceProvider(state: Pick<VoiceInputState, 'catalog'>): VoiceProviderStatus | undefined {
const catalog = state.catalog
return catalog?.providers.find(item => item.info.id === catalog.preferences.providerId)
}
/** True when the composer should offer the dictation button as usable. */
export function selectVoiceInputReady(state: Pick<VoiceInputState, 'catalog'>): boolean {
const catalog = state.catalog
if (!catalog?.supported || !catalog.preferences.enabled) return false
return selectActiveVoiceProvider(state)?.preparation.phase === 'ready'
}
@@ -18,6 +18,12 @@ const DEFAULT_PET_PREFERENCES = {
lastSessionId: null,
}
const DEFAULT_VOICE_INPUT_PREFERENCES = {
enabled: false,
providerId: 'sensevoice-local',
language: 'auto',
}
async function setup() {
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'desktop-ui-preferences-'))
originalConfigDir = process.env.CLAUDE_CONFIG_DIR
@@ -66,7 +72,7 @@ describe('DesktopUiPreferencesService', () => {
expect(result.exists).toBe(false)
expect(result.preferences).toEqual({
schemaVersion: 5,
schemaVersion: 6,
profile: {
displayName: 'cc-haha',
subtitle: 'github.com/NanmiCoder/cc-haha',
@@ -74,6 +80,7 @@ describe('DesktopUiPreferencesService', () => {
avatarUpdatedAt: null,
},
pet: DEFAULT_PET_PREFERENCES,
voiceInput: DEFAULT_VOICE_INPUT_PREFERENCES,
projectDisplayNames: {},
sidebar: {
projectOrder: [],
@@ -120,7 +127,7 @@ describe('DesktopUiPreferencesService', () => {
expect(before.exists).toBe(true)
expect(before.preferences).toEqual({
schemaVersion: 5,
schemaVersion: 6,
futureField: { keep: true },
profile: {
displayName: 'cc-haha',
@@ -129,6 +136,7 @@ describe('DesktopUiPreferencesService', () => {
avatarUpdatedAt: null,
},
pet: DEFAULT_PET_PREFERENCES,
voiceInput: DEFAULT_VOICE_INPUT_PREFERENCES,
projectDisplayNames: {
'/workspace/alpha': 'Alpha project',
},
@@ -141,7 +149,7 @@ describe('DesktopUiPreferencesService', () => {
},
})
expect(after).toEqual({
schemaVersion: 5,
schemaVersion: 6,
futureField: { keep: true },
profile: {
displayName: 'cc-haha',
@@ -150,6 +158,7 @@ describe('DesktopUiPreferencesService', () => {
avatarUpdatedAt: null,
},
pet: DEFAULT_PET_PREFERENCES,
voiceInput: DEFAULT_VOICE_INPUT_PREFERENCES,
projectDisplayNames: {
'/workspace/alpha': 'Alpha project',
},
@@ -307,7 +316,7 @@ describe('DesktopUiPreferencesService', () => {
})
expect(after).toMatchObject({
schemaVersion: 5,
schemaVersion: 6,
futureField: { keep: true },
profile: {
displayName: 'Local Operator',
@@ -386,7 +395,7 @@ describe('DesktopUiPreferencesService', () => {
const after = await new DesktopUiPreferencesService().updatePetPreferences({ enabled: true })
expect(after).toMatchObject({
schemaVersion: 5,
schemaVersion: 6,
futureField: { keep: true },
pet: {
futurePetField: { keep: 'pet-too' },
@@ -482,7 +491,7 @@ describe('DesktopUiPreferencesService', () => {
})
expect(after).toEqual({
schemaVersion: 5,
schemaVersion: 6,
profile: {
displayName: 'Claude Captain',
subtitle: 'local.example/profile',
@@ -490,6 +499,7 @@ describe('DesktopUiPreferencesService', () => {
avatarUpdatedAt: null,
},
pet: DEFAULT_PET_PREFERENCES,
voiceInput: DEFAULT_VOICE_INPUT_PREFERENCES,
projectDisplayNames: {},
sidebar: {
projectOrder: [],
@@ -575,7 +585,7 @@ describe('desktop UI preferences API', () => {
expect(putBody).toEqual({
ok: true,
preferences: {
schemaVersion: 5,
schemaVersion: 6,
profile: {
displayName: 'cc-haha',
subtitle: 'github.com/NanmiCoder/cc-haha',
@@ -583,6 +593,7 @@ describe('desktop UI preferences API', () => {
avatarUpdatedAt: null,
},
pet: DEFAULT_PET_PREFERENCES,
voiceInput: DEFAULT_VOICE_INPUT_PREFERENCES,
projectDisplayNames: {},
sidebar: {
projectOrder: ['/workspace/beta', '/workspace/alpha'],
@@ -602,7 +613,7 @@ describe('desktop UI preferences API', () => {
expect(getBody).toEqual({
exists: true,
preferences: {
schemaVersion: 5,
schemaVersion: 6,
profile: {
displayName: 'cc-haha',
subtitle: 'github.com/NanmiCoder/cc-haha',
@@ -610,6 +621,7 @@ describe('desktop UI preferences API', () => {
avatarUpdatedAt: null,
},
pet: DEFAULT_PET_PREFERENCES,
voiceInput: DEFAULT_VOICE_INPUT_PREFERENCES,
projectDisplayNames: {},
sidebar: {
projectOrder: ['/workspace/beta', '/workspace/alpha'],
@@ -700,7 +712,7 @@ describe('desktop UI preferences API', () => {
await expect(putRes.json()).resolves.toMatchObject({
ok: true,
preferences: {
schemaVersion: 5,
schemaVersion: 6,
pet: {
enabled: true,
selectedPetId: 'seedy',
@@ -939,3 +951,111 @@ describe('desktop UI preferences API', () => {
})
})
})
describe('DesktopUiPreferencesService voiceInput section', () => {
beforeEach(setup)
afterEach(teardown)
async function writeDesktopUiFile(contents: Record<string, unknown>) {
await fs.mkdir(path.join(tmpDir, 'cc-haha'), { recursive: true })
await fs.writeFile(path.join(tmpDir, 'cc-haha', 'desktop-ui.json'), JSON.stringify(contents), 'utf-8')
}
test('normalizes old schema files without voiceInput and preserves unknown fields', async () => {
await writeDesktopUiFile({
schemaVersion: 5,
futureField: { keep: true },
pet: { ...DEFAULT_PET_PREFERENCES, enabled: true },
sidebar: { projectOrder: ['/workspace/alpha'] },
})
const service = new DesktopUiPreferencesService()
const read = await service.readPreferences()
expect(read.exists).toBe(true)
expect(read.preferences).toMatchObject({
schemaVersion: 6,
futureField: { keep: true },
pet: { enabled: true },
voiceInput: DEFAULT_VOICE_INPUT_PREFERENCES,
})
const after = await service.updateVoiceInputPreferences({ enabled: true, language: 'zh' })
expect(after.voiceInput).toEqual({ enabled: true, providerId: 'sensevoice-local', language: 'zh' })
expect(await readDesktopUiFile()).toMatchObject({
schemaVersion: 6,
futureField: { keep: true },
pet: { enabled: true },
sidebar: { projectOrder: ['/workspace/alpha'] },
voiceInput: { enabled: true, providerId: 'sensevoice-local', language: 'zh' },
})
})
test('applies partial voice updates and keeps unknown voiceInput fields', async () => {
await writeDesktopUiFile({
schemaVersion: 6,
voiceInput: {
enabled: true,
providerId: 'cloud-x',
language: 'ja',
futureVoiceField: { keep: 'voice' },
},
})
const service = new DesktopUiPreferencesService()
const after = await service.updateVoiceInputPreferences({ language: 'ko' })
expect(after.voiceInput).toEqual({
enabled: true,
providerId: 'cloud-x',
language: 'ko',
futureVoiceField: { keep: 'voice' },
} as typeof after.voiceInput)
expect((await service.readPreferences()).preferences.voiceInput).toEqual(after.voiceInput)
})
test('replaces malformed voiceInput values with defaults', async () => {
await writeDesktopUiFile({
schemaVersion: 6,
voiceInput: { enabled: 'yes', providerId: ' ', language: 'fr' },
})
const service = new DesktopUiPreferencesService()
expect((await service.readPreferences()).preferences.voiceInput).toEqual(DEFAULT_VOICE_INPUT_PREFERENCES)
await writeDesktopUiFile({ schemaVersion: 6, voiceInput: 'garbage' })
expect((await service.readPreferences()).preferences.voiceInput).toEqual(DEFAULT_VOICE_INPUT_PREFERENCES)
const after = await service.updateVoiceInputPreferences({ enabled: 1, providerId: 'x'.repeat(200), language: 'zh' })
expect(after.voiceInput).toEqual({ ...DEFAULT_VOICE_INPUT_PREFERENCES, language: 'zh' })
})
test('other preference updates keep the saved voiceInput section', async () => {
const service = new DesktopUiPreferencesService()
await service.updateVoiceInputPreferences({ enabled: true, language: 'en' })
await service.updatePetPreferences({ enabled: true })
await service.updateSidebarPreferences({ pinnedProjects: ['/workspace/alpha'] })
await service.updateProfilePreferences({ displayName: 'Operator' })
await service.updateProjectDisplayName({ projectKey: '/workspace/alpha', displayName: 'Alpha' })
await service.clearProfileAvatar()
expect((await readDesktopUiFile()).voiceInput).toEqual({
enabled: true,
providerId: 'sensevoice-local',
language: 'en',
})
})
test('keeps a newer schemaVersion while filling in voiceInput defaults', async () => {
await writeDesktopUiFile({ schemaVersion: 99, futureField: 1 })
const read = await new DesktopUiPreferencesService().readPreferences()
expect(read.preferences).toMatchObject({
schemaVersion: 99,
futureField: 1,
voiceInput: DEFAULT_VOICE_INPUT_PREFERENCES,
})
})
})
+293
View File
@@ -0,0 +1,293 @@
import { afterAll, afterEach, beforeEach, describe, expect, test } from 'bun:test'
import * as fs from 'node:fs/promises'
import * as os from 'node:os'
import * as path from 'node:path'
import { handleApiRequest } from '../router.js'
import { handleVoiceApi } from '../api/voice.js'
import { createVoiceService } from '../services/voice/defaultRegistry.js'
import { VOICE_LIMITS, type VoiceCatalog, type VoiceProviderStatus, type VoiceTranscript } from '../services/voice/types.js'
import type { VoiceService } from '../services/voice/voiceService.js'
import { FakeProvider, makeWav } from '../services/voice/__tests__/fakeProvider.js'
let tmpDir: string
let originalConfigDir: string | undefined
let alpha: FakeProvider
let beta: FakeProvider
let service: VoiceService
const flush = () => new Promise<void>(resolve => setTimeout(resolve, 0))
beforeEach(async () => {
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'voice-api-'))
originalConfigDir = process.env.CLAUDE_CONFIG_DIR
process.env.CLAUDE_CONFIG_DIR = tmpDir
alpha = new FakeProvider({ id: 'sensevoice-local', name: 'Alpha', text: 'alpha text' })
beta = new FakeProvider({ id: 'beta', name: 'Beta', languages: ['auto', 'en'], text: 'beta text' })
service = createVoiceService([alpha, beta])
})
afterEach(async () => {
if (originalConfigDir !== undefined) process.env.CLAUDE_CONFIG_DIR = originalConfigDir
else delete process.env.CLAUDE_CONFIG_DIR
await fs.rm(tmpDir, { recursive: true, force: true })
})
async function call(method: string, target: string, body?: BodyInit | object, contentType?: string) {
const url = new URL(target, 'http://localhost:3456')
const init: RequestInit = { method }
if (body !== undefined) {
const raw = body instanceof Uint8Array || typeof body === 'string' ? body : JSON.stringify(body)
init.body = raw as BodyInit
init.headers = { 'Content-Type': contentType ?? (body instanceof Uint8Array ? 'audio/wav' : 'application/json') }
}
const req = new Request(url, init)
return handleVoiceApi(req, url, url.pathname.split('/').filter(Boolean), service)
}
describe('GET /api/voice/catalog', () => {
test('returns the catalog shape', async () => {
const res = await call('GET', '/api/voice/catalog')
expect(res.status).toBe(200)
const catalog = await res.json() as VoiceCatalog
expect(catalog).toEqual({
supported: true,
providers: [
{
info: { id: 'sensevoice-local', name: 'Alpha', location: 'local', languages: ['auto', 'zh', 'en'], downloadBytes: 1000 },
preparation: { phase: 'unprepared' },
},
{
info: { id: 'beta', name: 'Beta', location: 'local', languages: ['auto', 'en'], downloadBytes: 1000 },
preparation: { phase: 'unprepared' },
},
],
preferences: { enabled: false, providerId: 'sensevoice-local', language: 'auto' },
limits: VOICE_LIMITS,
})
})
test('is reachable through the main API router', async () => {
const url = new URL('http://localhost:3456/api/voice/catalog')
const res = await handleApiRequest(new Request(url), url)
expect(res.status).toBe(200)
const catalog = await res.json() as VoiceCatalog
expect(Object.keys(catalog).sort()).toEqual(['limits', 'preferences', 'providers', 'supported'])
})
test('rejects other methods', async () => {
expect((await call('POST', '/api/voice/catalog', {})).status).toBe(405)
})
})
describe('PUT /api/voice/preferences', () => {
test('persists a partial update to desktop-ui.json and keeps other sections', async () => {
await fs.mkdir(path.join(tmpDir, 'cc-haha'), { recursive: true })
await fs.writeFile(
path.join(tmpDir, 'cc-haha', 'desktop-ui.json'),
JSON.stringify({ schemaVersion: 6, futureField: { keep: true }, pet: { enabled: true } }),
)
const first = await call('PUT', '/api/voice/preferences', { enabled: true })
expect(first.status).toBe(200)
expect(await first.json()).toEqual({
preferences: { enabled: true, providerId: 'sensevoice-local', language: 'auto' },
})
const second = await call('PUT', '/api/voice/preferences', { language: 'zh' })
expect(await second.json()).toEqual({
preferences: { enabled: true, providerId: 'sensevoice-local', language: 'zh' },
})
const file = JSON.parse(await fs.readFile(path.join(tmpDir, 'cc-haha', 'desktop-ui.json'), 'utf-8'))
expect(file.voiceInput).toEqual({ enabled: true, providerId: 'sensevoice-local', language: 'zh' })
expect(file.futureField).toEqual({ keep: true })
expect(file.pet.enabled).toBe(true)
const catalog = await (await call('GET', '/api/voice/catalog')).json() as VoiceCatalog
expect(catalog.preferences.language).toBe('zh')
})
test('rejects unknown providers with 404 and invalid values with 400, writing nothing', async () => {
const unknown = await call('PUT', '/api/voice/preferences', { providerId: 'nope' })
expect(unknown.status).toBe(404)
expect(await unknown.json()).toMatchObject({ error: 'voice/unknown-provider' })
expect((await call('PUT', '/api/voice/preferences', { language: 'fr' })).status).toBe(400)
expect((await call('PUT', '/api/voice/preferences', { providerId: 'beta', language: 'zh' })).status).toBe(400)
expect((await call('PUT', '/api/voice/preferences', { enabled: 'yes' })).status).toBe(400)
expect((await call('PUT', '/api/voice/preferences', 'not json', 'application/json')).status).toBe(400)
await expect(fs.access(path.join(tmpDir, 'cc-haha', 'desktop-ui.json'))).rejects.toThrow()
})
})
describe('provider endpoints', () => {
test('prepare returns immediately, status tracks progress, cancel and delete round-trip', async () => {
const prepare = await call('POST', '/api/voice/providers/sensevoice-local/prepare')
expect(prepare.status).toBe(200)
expect((await prepare.json() as VoiceProviderStatus).preparation.phase).toBe('downloading')
await alpha.started
alpha.report({ phase: 'downloading', step: 'model', completedBytes: 5, totalBytes: 10 })
const status = await (await call('GET', '/api/voice/providers/sensevoice-local/status')).json() as VoiceProviderStatus
expect(status.preparation).toMatchObject({ completedBytes: 5, totalBytes: 10 })
const cancel = await (await call('POST', '/api/voice/providers/sensevoice-local/cancel')).json() as VoiceProviderStatus
expect(cancel.preparation.phase).toBe('cancelled')
await flush()
alpha.installed = true
const removed = await call('DELETE', '/api/voice/providers/sensevoice-local/assets')
expect((await removed.json() as VoiceProviderStatus).preparation.phase).toBe('unprepared')
expect(alpha.removeCalls).toBe(1)
})
test.each([
['GET', '/api/voice/providers/nope/status'],
['POST', '/api/voice/providers/nope/prepare'],
['POST', '/api/voice/providers/nope/cancel'],
['DELETE', '/api/voice/providers/nope/assets'],
])('%s %s returns 404 for an unknown provider', async (method, target) => {
const res = await call(method, target)
expect(res.status).toBe(404)
expect(await res.json()).toMatchObject({ error: 'voice/unknown-provider' })
})
test('treats a malformed percent-encoded provider id as an unknown provider, not a server error', async () => {
// decodeURIComponent throws URIError on `%E0%A4%A`; that must not become a 500.
for (const action of ['status', 'prepare', 'cancel']) {
const method = action === 'status' ? 'GET' : 'POST'
const res = await call(method, `/api/voice/providers/%E0%A4%A/${action}`)
expect(res.status).toBe(404)
expect(await res.json()).toMatchObject({ error: 'voice/unknown-provider' })
}
})
test('rejects wrong methods and unknown actions', async () => {
expect((await call('GET', '/api/voice/providers/beta/prepare')).status).toBe(405)
expect((await call('GET', '/api/voice/providers/beta/bogus')).status).toBe(404)
expect((await call('GET', '/api/voice/providers/beta')).status).toBe(404)
expect((await call('GET', '/api/voice/bogus')).status).toBe(404)
})
})
describe('POST /api/voice/transcribe', () => {
test('returns a transcript for a ready provider using saved preferences', async () => {
alpha.installed = true
await call('PUT', '/api/voice/preferences', { language: 'zh' })
const res = await call('POST', '/api/voice/transcribe', makeWav(2))
expect(res.status).toBe(200)
expect(await res.json() as VoiceTranscript).toEqual({ text: 'alpha text', audioSeconds: 2, inferenceSeconds: 0.25 })
expect(alpha.transcribeCalls).toEqual([{ bytes: makeWav(2).byteLength, language: 'zh' }])
})
test('honours the provider and language query, even while the feature is disabled', async () => {
beta.installed = true
const res = await call('POST', '/api/voice/transcribe?provider=beta&language=en', makeWav(1))
expect(res.status).toBe(200)
expect((await res.json() as VoiceTranscript).text).toBe('beta text')
expect(beta.transcribeCalls[0]!.language).toBe('en')
})
test('returns 409 voice/not-ready before assets are installed', async () => {
const res = await call('POST', '/api/voice/transcribe', makeWav(1))
expect(res.status).toBe(409)
expect(await res.json()).toMatchObject({ error: 'voice/not-ready' })
})
test('returns 404 for an unknown provider', async () => {
const res = await call('POST', '/api/voice/transcribe?provider=nope', makeWav(1))
expect(res.status).toBe(404)
expect(await res.json()).toMatchObject({ error: 'voice/unknown-provider' })
})
test('returns 400 for a language the provider does not offer or a nonsense code', async () => {
beta.installed = true
expect((await call('POST', '/api/voice/transcribe?provider=beta&language=ja', makeWav(1))).status).toBe(400)
expect((await call('POST', '/api/voice/transcribe?provider=beta&language=klingon', makeWav(1))).status).toBe(400)
})
test('returns 400 voice/invalid-audio for garbage, empty and over-limit audio', async () => {
alpha.installed = true
for (const body of [
new Uint8Array(200),
new Uint8Array(0),
makeWav(VOICE_LIMITS.maxAudioSeconds + 1),
new Uint8Array(VOICE_LIMITS.maxAudioBytes + 1),
]) {
const res = await call('POST', '/api/voice/transcribe', body)
expect(res.status).toBe(400)
expect(await res.json()).toMatchObject({ error: 'voice/invalid-audio' })
}
expect(alpha.transcribeCalls).toHaveLength(0)
})
test('returns 500 voice/failed when the provider throws', async () => {
alpha.installed = true
alpha.transcribeError = new Error('worker crashed')
const res = await call('POST', '/api/voice/transcribe', makeWav(1))
expect(res.status).toBe(500)
expect(await res.json()).toMatchObject({ error: 'voice/failed', message: expect.stringContaining('worker crashed') })
})
})
describe('transcribe over a real HTTP server', () => {
const servers: Array<ReturnType<typeof Bun.serve>> = []
afterAll(() => {
for (const server of servers) void server.stop(true)
})
function serve() {
const server = Bun.serve({
port: 0,
hostname: '127.0.0.1',
fetch(req) {
const url = new URL(req.url)
return handleVoiceApi(req, url, url.pathname.split('/').filter(Boolean), service)
},
})
servers.push(server)
return `http://127.0.0.1:${server.port}`
}
test('round-trips a near-limit binary body and rejects an oversized chunked body', async () => {
alpha.installed = true
const base = serve()
// 120 s of audio is 3.84 MB, above the 4 MB default some frameworks cap at.
const wav = makeWav(VOICE_LIMITS.maxAudioSeconds)
const ok = await fetch(`${base}/api/voice/transcribe`, { method: 'POST', headers: { 'Content-Type': 'audio/wav' }, body: wav })
expect(ok.status).toBe(200)
expect(alpha.transcribeCalls[0]!.bytes).toBe(wav.byteLength)
const oversized = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array(VOICE_LIMITS.maxAudioBytes))
controller.enqueue(new Uint8Array(1024))
controller.close()
},
})
const rejected = await fetch(`${base}/api/voice/transcribe`, {
method: 'POST',
headers: { 'Content-Type': 'audio/wav' },
body: oversized,
duplex: 'half',
} as RequestInit)
expect(rejected.status).toBe(400)
expect(await rejected.json()).toMatchObject({ error: 'voice/invalid-audio' })
})
})
+148
View File
@@ -0,0 +1,148 @@
/**
* Voice input REST API
*
* GET /api/voice/catalog — providers, preferences, limits
* PUT /api/voice/preferences — partial update of { enabled, providerId, language }
* POST /api/voice/providers/:id/prepare — start (or join) the background download
* POST /api/voice/providers/:id/cancel — cancel a running download
* GET /api/voice/providers/:id/status — provider info + preparation state
* DELETE /api/voice/providers/:id/assets — delete downloaded runtime and models
* POST /api/voice/transcribe?provider=&language= — raw 16 kHz mono PCM16 WAV body
*
* Voice errors use `{ error: 'voice/...', message }` with 400/404/409/500.
*/
import { ApiError, errorResponse } from '../middleware/errorHandler.js'
import { getVoiceService } from '../services/voice/defaultRegistry.js'
import { VoiceServiceError } from '../services/voice/errors.js'
import { isVoiceLanguage } from '../services/voice/preferences.js'
import { VOICE_LIMITS } from '../services/voice/types.js'
import type { VoiceService } from '../services/voice/voiceService.js'
export async function handleVoiceApi(
req: Request,
url: URL,
segments: string[],
service: VoiceService = getVoiceService(),
): Promise<Response> {
try {
const sub = segments[2]
if (sub === 'catalog') {
requireMethod(req, 'GET')
return Response.json(await service.catalog())
}
if (sub === 'preferences') {
requireMethod(req, 'PUT')
const preferences = await service.updatePreferences(await parseJsonBody(req))
return Response.json({ preferences })
}
if (sub === 'providers') {
const providerId = segments[3] ? decodeProviderId(segments[3]) : undefined
const action = segments[4]
if (!providerId || !action) throw ApiError.notFound('Unknown voice provider endpoint')
if (action === 'status') {
requireMethod(req, 'GET')
return Response.json(await service.status(providerId))
}
if (action === 'prepare') {
requireMethod(req, 'POST')
return Response.json(await service.prepare(providerId))
}
if (action === 'cancel') {
requireMethod(req, 'POST')
return Response.json(await service.cancel(providerId))
}
if (action === 'assets') {
requireMethod(req, 'DELETE')
return Response.json(await service.removeAssets(providerId))
}
throw ApiError.notFound(`Unknown voice provider endpoint: ${action}`)
}
if (sub === 'transcribe') {
requireMethod(req, 'POST')
const language = url.searchParams.get('language') ?? undefined
if (language !== undefined && !isVoiceLanguage(language)) {
throw ApiError.badRequest(`Unsupported language: ${language}`)
}
const wav = await readAudioBody(req)
const transcript = await service.transcribe(
url.searchParams.get('provider') ?? undefined,
wav,
language,
req.signal,
)
return Response.json(transcript)
}
throw ApiError.notFound(`Unknown voice endpoint: ${sub}`)
} catch (error) {
if (error instanceof VoiceServiceError) {
return Response.json(error.toBody(), { status: error.status })
}
return errorResponse(error)
}
}
/** A malformed `%` escape can never name a registered provider; answer 404 rather than throwing a URIError. */
function decodeProviderId(segment: string): string {
try {
return decodeURIComponent(segment)
} catch {
throw new VoiceServiceError('voice/unknown-provider', 'Unknown voice provider')
}
}
/**
* Reads the upload without buffering past the audio limit. The main server
* accepts bodies far larger than a dictation clip, so the cap lives here.
*/
async function readAudioBody(req: Request): Promise<Uint8Array> {
const max = VOICE_LIMITS.maxAudioBytes
const tooLarge = () =>
new VoiceServiceError('voice/invalid-audio', `Audio is too large (limit ${max} bytes)`)
const declared = Number(req.headers.get('Content-Length'))
if (Number.isFinite(declared) && declared > max) throw tooLarge()
if (!req.body) throw new VoiceServiceError('voice/invalid-audio', 'Audio body is empty')
const chunks: Uint8Array[] = []
let total = 0
const reader = req.body.getReader()
for (;;) {
const { done, value } = await reader.read()
if (done) break
total += value.byteLength
if (total > max) {
await reader.cancel().catch(() => {})
throw tooLarge()
}
chunks.push(value)
}
const bytes = new Uint8Array(total)
let offset = 0
for (const chunk of chunks) {
bytes.set(chunk, offset)
offset += chunk.byteLength
}
return bytes
}
async function parseJsonBody(req: Request): Promise<unknown> {
try {
return await req.json()
} catch {
throw ApiError.badRequest('Invalid JSON body')
}
}
function requireMethod(req: Request, method: string): void {
if (req.method !== method) {
throw new ApiError(405, `Method ${req.method} not allowed`, 'METHOD_NOT_ALLOWED')
}
}
+4
View File
@@ -32,6 +32,7 @@ import { handleMemoryApi } from './api/memory.js'
import { handleDesktopUiApi } from './api/desktop-ui.js'
import { handleTracesApi } from './api/traces.js'
import { handleWorkflowsApi } from './api/workflows.js'
import { handleVoiceApi } from './api/voice.js'
import { apiPerformanceMonitor } from './services/apiPerformanceMonitor.js'
import { remoteProviderRouteAllowed, remoteSettingsRouteAllowed, projectRemoteProvider, projectRemoteSettings, replaceRemoteCompatibility, validateRemoteSettingsPatch, type ApiRequestContext } from './remoteBrowserPolicy.js'
@@ -207,6 +208,9 @@ async function routeApiRequest(req: Request, url: URL): Promise<Response> {
case 'desktop-ui':
return handleDesktopUiApi(req, url, segments)
case 'voice':
return handleVoiceApi(req, url, segments)
case 'traces':
return handleTracesApi(req, url, segments)
@@ -5,8 +5,10 @@ import { randomBytes } from 'node:crypto'
import { ApiError } from '../middleware/errorHandler.js'
import { readRecoverableJsonFile } from './recoverableJsonFile.js'
import { ensurePersistentStorageUpgraded } from './persistentStorageMigrations.js'
import { normalizeVoicePreferences } from './voice/preferences.js'
import { DEFAULT_VOICE_PREFERENCES, type VoicePreferences } from './voice/types.js'
const CURRENT_DESKTOP_UI_PREFERENCES_SCHEMA_VERSION = 5
const CURRENT_DESKTOP_UI_PREFERENCES_SCHEMA_VERSION = 6
const MAX_PROJECT_PREFERENCE_ENTRIES = 2_000
const MAX_PROJECT_DISPLAY_NAME_ENTRIES = 2_000
const MAX_PROJECT_DISPLAY_NAME_KEY_LENGTH = 4_096
@@ -53,6 +55,8 @@ export type DesktopPetPreferences = {
lastSessionId: string | null
}
export type DesktopVoiceInputPreferences = VoicePreferences
export type ProjectDisplayNames = Record<string, string>
export type DesktopUiPreferences = {
@@ -60,6 +64,7 @@ export type DesktopUiPreferences = {
sidebar: SidebarProjectPreferences
profile: DesktopProfilePreferences
pet: DesktopPetPreferences
voiceInput: DesktopVoiceInputPreferences
projectDisplayNames: ProjectDisplayNames
[key: string]: unknown
}
@@ -104,6 +109,7 @@ function defaultPreferences(): DesktopUiPreferences {
sidebar: { ...DEFAULT_SIDEBAR_PROJECT_PREFERENCES },
profile: { ...DEFAULT_PROFILE_PREFERENCES },
pet: { ...DEFAULT_PET_PREFERENCES },
voiceInput: { ...DEFAULT_VOICE_PREFERENCES },
projectDisplayNames: createProjectDisplayNames(),
}
}
@@ -342,6 +348,7 @@ function normalizeDesktopUiPreferences(value: unknown): DesktopUiPreferences | n
sidebar: normalizeSidebarProjectPreferences(record.sidebar),
profile: normalizeProfilePreferences(record.profile),
pet: normalizeDesktopPetPreferences(record.pet),
voiceInput: normalizeVoicePreferences(record.voiceInput),
projectDisplayNames: normalizeProjectDisplayNames(record.projectDisplayNames),
}
}
@@ -455,6 +462,7 @@ export class DesktopUiPreferencesService {
}),
profile: normalizeProfilePreferences(preferences.profile),
pet: normalizeDesktopPetPreferences(preferences.pet),
voiceInput: normalizeVoicePreferences(preferences.voiceInput),
projectDisplayNames: normalizeProjectDisplayNames(preferences.projectDisplayNames),
}
@@ -488,6 +496,7 @@ export class DesktopUiPreferencesService {
avatarUpdatedAt: currentProfile.avatarUpdatedAt,
},
pet: normalizeDesktopPetPreferences(preferences.pet),
voiceInput: normalizeVoicePreferences(preferences.voiceInput),
projectDisplayNames: normalizeProjectDisplayNames(preferences.projectDisplayNames),
}
@@ -525,6 +534,7 @@ export class DesktopUiPreferencesService {
sidebar: normalizeSidebarProjectPreferences(preferences.sidebar),
profile: normalizeProfilePreferences(preferences.profile),
pet: normalizeDesktopPetPreferences(preferences.pet),
voiceInput: normalizeVoicePreferences(preferences.voiceInput),
projectDisplayNames,
}
@@ -550,6 +560,33 @@ export class DesktopUiPreferencesService {
...currentPet,
...patch,
}),
voiceInput: normalizeVoicePreferences(preferences.voiceInput),
projectDisplayNames: normalizeProjectDisplayNames(preferences.projectDisplayNames),
}
await this.writePreferences(nextPreferences)
return nextPreferences
})
}
async updateVoiceInputPreferences(voiceInput: unknown): Promise<DesktopUiPreferences> {
const filePath = this.getPreferencesPath()
return this.withWriteLock(filePath, async () => {
const { preferences } = await this.readPreferences()
const currentVoiceInput = normalizeVoicePreferences(preferences.voiceInput)
const patch = voiceInput && typeof voiceInput === 'object' && !Array.isArray(voiceInput)
? voiceInput as Record<string, unknown>
: {}
const nextPreferences: DesktopUiPreferences = {
...preferences,
schemaVersion: preferences.schemaVersion,
sidebar: normalizeSidebarProjectPreferences(preferences.sidebar),
profile: normalizeProfilePreferences(preferences.profile),
pet: normalizeDesktopPetPreferences(preferences.pet),
voiceInput: normalizeVoicePreferences({
...currentVoiceInput,
...patch,
}),
projectDisplayNames: normalizeProjectDisplayNames(preferences.projectDisplayNames),
}
@@ -601,6 +638,7 @@ export class DesktopUiPreferencesService {
avatarUpdatedAt: new Date().toISOString(),
},
pet: normalizeDesktopPetPreferences(preferences.pet),
voiceInput: normalizeVoicePreferences(preferences.voiceInput),
projectDisplayNames: normalizeProjectDisplayNames(preferences.projectDisplayNames),
}
@@ -628,6 +666,7 @@ export class DesktopUiPreferencesService {
avatarUpdatedAt: null,
},
pet: normalizeDesktopPetPreferences(preferences.pet),
voiceInput: normalizeVoicePreferences(preferences.voiceInput),
projectDisplayNames: normalizeProjectDisplayNames(preferences.projectDisplayNames),
}
@@ -0,0 +1,85 @@
import { describe, expect, test } from 'bun:test'
import * as fs from 'node:fs'
import * as path from 'node:path'
import { VOICE_LANGUAGES } from '../preferences.js'
import { VOICE_LIMITS } from '../types.js'
import { makeWav } from './fakeProvider.js'
// The desktop renderer keeps a hand-written mirror of the server contract. These
// tests read both source files and compare them so the two cannot drift apart.
const repoRoot = path.resolve(import.meta.dir, '../../../../..')
const serverSource = fs.readFileSync(path.join(repoRoot, 'src/server/services/voice/types.ts'), 'utf-8')
const desktopSource = fs.readFileSync(path.join(repoRoot, 'desktop/src/api/voice.ts'), 'utf-8')
const desktopPreferencesSource = fs.readFileSync(path.join(repoRoot, 'desktop/src/api/desktopUiPreferences.ts'), 'utf-8')
function stripComments(source: string): string {
return source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^\s*\/\/.*$/gm, '')
}
function literalUnion(source: string, name: string): string[] {
const text = stripComments(source)
const start = text.indexOf(`export type ${name} =`)
if (start < 0) throw new Error(`type ${name} not found`)
const rest = text.slice(start)
const end = rest.search(/\n\s*\n|\nexport /)
const literals = [...(end < 0 ? rest : rest.slice(0, end)).matchAll(/'([^']+)'/g)].map(item => item[1]!)
if (literals.length === 0) throw new Error(`type ${name} has no string literals`)
return literals.sort()
}
function objectFields(source: string, name: string): Record<string, string> {
const match = stripComments(source).match(
new RegExp(`export (?:interface ${name} |type ${name} = )\\{([\\s\\S]*?)\\n\\}`),
)
if (!match) throw new Error(`object type ${name} not found`)
const fields: Record<string, string> = {}
for (const line of match[1]!.split('\n')) {
const field = line.trim().match(/^([A-Za-z]+\??):\s*(.+)$/)
if (field) fields[field[1]!] = field[2]!.replace(/\s+/g, ' ')
}
if (Object.keys(fields).length === 0) throw new Error(`object type ${name} has no fields`)
return fields
}
describe('voice contract parity (server types.ts vs desktop api/voice.ts)', () => {
test.each([
'VoiceLanguage',
'VoicePreparationPhase',
'VoicePreparationStep',
'VoiceFailureReason',
'VoiceErrorCode',
])('%s has the same literals on both sides', name => {
expect(literalUnion(desktopSource, name)).toEqual(literalUnion(serverSource, name))
})
test.each([
'VoiceFailure',
'VoicePreparationState',
'VoiceProviderInfo',
'VoiceProviderStatus',
'VoicePreferences',
'VoiceLimits',
'VoiceCatalog',
'VoiceTranscript',
])('%s has the same fields and field types on both sides', name => {
expect(objectFields(desktopSource, name)).toEqual(objectFields(serverSource, name))
})
test('the runtime language list used to validate preferences matches the VoiceLanguage type', () => {
expect([...VOICE_LANGUAGES].sort()).toEqual(literalUnion(serverSource, 'VoiceLanguage'))
})
test('the desktop client takes upload limits from the catalog instead of redefining them', () => {
expect(stripComments(desktopSource)).not.toMatch(/maxAudio(?:Seconds|Bytes)\s*[:=]\s*\d/)
expect(stripComments(desktopSource)).not.toContain('VOICE_LIMITS')
})
test('the desktop preferences mirror carries the voiceInput section', () => {
expect(objectFields(desktopPreferencesSource, 'DesktopUiPreferences')['voiceInput?']).toBe('VoicePreferences')
})
test('the default limits admit a full-length canonical recording', () => {
expect(VOICE_LIMITS.maxAudioSeconds).toBe(120)
expect(makeWav(VOICE_LIMITS.maxAudioSeconds).byteLength).toBeLessThanOrEqual(VOICE_LIMITS.maxAudioBytes)
})
})
@@ -0,0 +1,42 @@
import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
import * as fs from 'node:fs/promises'
import * as os from 'node:os'
import * as path from 'node:path'
import { createDefaultProviders, createVoiceService } from '../defaultRegistry.js'
import { DEFAULT_VOICE_PREFERENCES } from '../types.js'
let configDir: string
let previousConfigDir: string | undefined
beforeEach(async () => {
configDir = await fs.mkdtemp(path.join(os.tmpdir(), 'cc-haha-voice-default-'))
previousConfigDir = process.env.CLAUDE_CONFIG_DIR
process.env.CLAUDE_CONFIG_DIR = configDir
})
afterEach(async () => {
if (previousConfigDir === undefined) delete process.env.CLAUDE_CONFIG_DIR
else process.env.CLAUDE_CONFIG_DIR = previousConfigDir
await fs.rm(configDir, { recursive: true, force: true })
})
describe('default voice providers', () => {
test('ships the local SenseVoice provider under the default preference id', () => {
const ids = createDefaultProviders().map(provider => provider.info.id)
expect(ids).toEqual([DEFAULT_VOICE_PREFERENCES.providerId])
})
test('reports an unprepared local provider without touching the network or the real home', async () => {
const service = createVoiceService(createDefaultProviders())
const catalog = await service.catalog()
expect(catalog.providers).toHaveLength(1)
const [sensevoice] = catalog.providers
expect(sensevoice!.info.location).toBe('local')
expect(sensevoice!.info.languages).toContain('zh')
expect(sensevoice!.preparation.phase).toBe('unprepared')
expect(catalog.preferences.providerId).toBe(sensevoice!.info.id)
// Preferences are read from the redirected config dir, never the real one.
await expect(fs.readdir(configDir)).resolves.toBeDefined()
})
})
@@ -0,0 +1,174 @@
import type {
SpeechProvider,
VoiceLanguage,
VoicePreferences,
VoicePreparationState,
VoiceProviderInfo,
} from '../types.js'
import { DEFAULT_VOICE_PREFERENCES } from '../types.js'
import type { VoicePreferencesStore } from '../preferencesStore.js'
export const SAMPLE_RATE = 16_000
const UNSUPPORTED_STATE = {
phase: 'failed',
error: { reason: 'unsupported-platform', message: 'not available on this platform' },
} as const satisfies VoicePreparationState
/** Builds a canonical 44-byte-header PCM16 mono 16 kHz WAV of the given duration. */
export function makeWav(seconds: number, overrides: {
channels?: number
sampleRate?: number
bits?: number
format?: number
dataLength?: number
riff?: string
wave?: string
} = {}): Uint8Array {
const dataBytes = Math.round(seconds * SAMPLE_RATE) * 2
const bytes = new Uint8Array(44 + dataBytes)
const view = new DataView(bytes.buffer)
const text = (offset: number, value: string) => {
for (let index = 0; index < 4; index += 1) bytes[offset + index] = value.charCodeAt(index)
}
text(0, overrides.riff ?? 'RIFF')
view.setUint32(4, 36 + dataBytes, true)
text(8, overrides.wave ?? 'WAVE')
text(12, 'fmt ')
view.setUint32(16, 16, true)
view.setUint16(20, overrides.format ?? 1, true)
view.setUint16(22, overrides.channels ?? 1, true)
view.setUint32(24, overrides.sampleRate ?? SAMPLE_RATE, true)
view.setUint32(28, SAMPLE_RATE * 2, true)
view.setUint16(32, 2, true)
view.setUint16(34, overrides.bits ?? 16, true)
text(36, 'data')
view.setUint32(40, overrides.dataLength ?? dataBytes, true)
return bytes
}
export type FakeProviderOptions = {
id: string
name?: string
languages?: VoiceLanguage[]
location?: VoiceProviderInfo['location']
/** Omit to model a provider that needs no local assets. */
needsAssets?: boolean
/** Models a provider on a platform it has no runtime for. */
unsupported?: boolean
text?: string
}
/**
* Deterministic provider. `prepare()` blocks until `finishPrepare()` or
* `failPrepare()` is called (or the signal aborts), so tests can observe the
* in-flight state.
*/
export class FakeProvider implements SpeechProvider {
readonly info: VoiceProviderInfo
readonly preparation?: SpeechProvider['preparation']
installed = false
prepareCalls = 0
removeCalls = 0
transcribeCalls: Array<{ bytes: number; language: VoiceLanguage }> = []
transcribeError?: Error
/** When set, an aborted prepare() keeps running until releaseAbort() is called. */
holdAbort = false
private abortRelease: Array<() => void> = []
private readonly text: string
private resolvePrepare?: () => void
private rejectPrepare?: (error: Error) => void
private reportFn?: (state: VoicePreparationState) => void
/** Resolves once prepare() has been entered. */
started: Promise<void>
private markStarted!: () => void
constructor(options: FakeProviderOptions) {
this.info = {
id: options.id,
name: options.name ?? options.id,
location: options.location ?? 'local',
languages: options.languages ?? ['auto', 'zh', 'en'],
downloadBytes: 1000,
}
this.text = options.text ?? `transcript from ${options.id}`
this.started = new Promise(resolve => { this.markStarted = resolve })
if (options.needsAssets !== false) {
this.preparation = {
status: async () => options.unsupported
? UNSUPPORTED_STATE
: this.installed ? { phase: 'ready' } : { phase: 'unprepared' },
prepare: options.unsupported
? async (_signal, report) => {
report(UNSUPPORTED_STATE)
throw new Error(UNSUPPORTED_STATE.error.message)
}
: (signal, report) => this.runPrepare(signal, report),
remove: async () => {
this.removeCalls += 1
this.installed = false
},
}
}
}
private runPrepare(signal: AbortSignal, report: (state: VoicePreparationState) => void): Promise<void> {
this.prepareCalls += 1
this.reportFn = report
this.markStarted()
return new Promise<void>((resolve, reject) => {
this.resolvePrepare = () => {
this.installed = true
resolve()
}
this.rejectPrepare = reject
signal.addEventListener('abort', () => {
if (this.holdAbort) this.abortRelease.push(() => reject(new Error('aborted')))
else reject(new Error('aborted'))
}, { once: true })
})
}
report(state: VoicePreparationState): void {
this.reportFn?.(state)
}
finishPrepare(): void {
this.resolvePrepare?.()
}
releaseAbort(): void {
for (const release of this.abortRelease.splice(0)) release()
}
failPrepare(error: Error): void {
this.rejectPrepare?.(error)
}
/** Makes the next prepare() cycle observable again. */
resetStarted(): void {
this.started = new Promise(resolve => { this.markStarted = resolve })
}
async transcribe(wav: Uint8Array, options: { language: VoiceLanguage }) {
this.transcribeCalls.push({ bytes: wav.byteLength, language: options.language })
if (this.transcribeError) throw this.transcribeError
return { text: this.text, audioSeconds: 0, inferenceSeconds: 0.25 }
}
}
export function memoryPreferencesStore(initial: Partial<VoicePreferences> = {}): VoicePreferencesStore & {
current: VoicePreferences
} {
const store = {
current: { ...DEFAULT_VOICE_PREFERENCES, ...initial } as VoicePreferences,
async read() {
return { ...store.current }
},
async update(patch: Partial<VoicePreferences>) {
store.current = { ...store.current, ...patch }
return { ...store.current }
},
}
return store
}
@@ -0,0 +1,33 @@
import { describe, expect, test } from 'bun:test'
import { VoiceProviderRegistry } from '../registry.js'
import { FakeProvider } from './fakeProvider.js'
describe('VoiceProviderRegistry', () => {
test('registers, looks up and lists providers in insertion order', () => {
const registry = new VoiceProviderRegistry()
const a = new FakeProvider({ id: 'a' })
const b = new FakeProvider({ id: 'b' })
registry.register(a).register(b)
expect(registry.get('a')).toBe(a)
expect(registry.get('b')).toBe(b)
expect(registry.list()).toEqual([a, b])
})
test('returns undefined for unknown ids without falling back to another provider', () => {
const registry = new VoiceProviderRegistry()
registry.register(new FakeProvider({ id: 'a' }))
expect(registry.get('missing')).toBeUndefined()
})
test('rejects a duplicate id and keeps the first registration', () => {
const registry = new VoiceProviderRegistry()
const first = new FakeProvider({ id: 'dup' })
registry.register(first)
expect(() => registry.register(new FakeProvider({ id: 'dup' }))).toThrow('already registered')
expect(registry.get('dup')).toBe(first)
expect(registry.list()).toHaveLength(1)
})
})
@@ -0,0 +1,464 @@
import { describe, expect, test } from 'bun:test'
import { ApiError } from '../../../middleware/errorHandler.js'
import { VoiceServiceError } from '../errors.js'
import { VoiceProviderRegistry } from '../registry.js'
import { VOICE_LIMITS } from '../types.js'
import { VoiceService } from '../voiceService.js'
import { FakeProvider, makeWav, memoryPreferencesStore } from './fakeProvider.js'
const flush = () => new Promise<void>(resolve => setTimeout(resolve, 0))
const signal = () => new AbortController().signal
function createService(
providers: FakeProvider[],
options: { preferences?: Parameters<typeof memoryPreferencesStore>[0] } = {},
) {
const registry = new VoiceProviderRegistry()
for (const provider of providers) registry.register(provider)
const preferences = memoryPreferencesStore({ providerId: providers[0]?.info.id, ...options.preferences })
const service = new VoiceService({ registry, preferences })
return { service, preferences }
}
async function expectRejects<T extends Error>(promise: Promise<unknown>, type: new (...args: never[]) => T) {
try {
await promise
} catch (error) {
expect(error).toBeInstanceOf(type)
return error as T
}
throw new Error('expected promise to reject')
}
describe('VoiceService catalog', () => {
test('lists every registered provider with preferences and limits', async () => {
const a = new FakeProvider({ id: 'alpha', name: 'Alpha' })
const b = new FakeProvider({ id: 'beta', name: 'Beta', languages: ['auto', 'ja'] })
b.installed = true
const { service } = createService([a, b], { preferences: { providerId: 'alpha' } })
const catalog = await service.catalog()
expect(catalog.supported).toBe(true)
expect(catalog.limits).toEqual(VOICE_LIMITS)
expect(catalog.preferences).toEqual({ enabled: false, providerId: 'alpha', language: 'auto' })
expect(catalog.providers.map(item => [item.info.id, item.preparation.phase])).toEqual([
['alpha', 'unprepared'],
['beta', 'ready'],
])
expect(catalog.providers[1]!.info.languages).toEqual(['auto', 'ja'])
})
test('is unsupported only when no provider exists or every provider reports unsupported-platform', async () => {
const { service: noRuntime } = createService([new FakeProvider({ id: 'alpha', unsupported: true })])
const { service: mixed } = createService([
new FakeProvider({ id: 'alpha', unsupported: true }),
new FakeProvider({ id: 'beta' }),
])
const { service: empty } = createService([])
const unsupported = await noRuntime.catalog()
expect(unsupported.supported).toBe(false)
expect(unsupported.providers[0]!.preparation).toMatchObject({
phase: 'failed',
error: { reason: 'unsupported-platform' },
})
expect((await mixed.catalog()).supported).toBe(true)
expect((await empty.catalog()).supported).toBe(false)
})
test('treats a provider without assets as ready', async () => {
const { service } = createService([new FakeProvider({ id: 'cloud', location: 'cloud', needsAssets: false })])
expect((await service.status('cloud')).preparation).toEqual({ phase: 'ready' })
})
})
describe.each([
['first provider', () => new FakeProvider({ id: 'alpha', text: 'hello from alpha' }), 'hello from alpha'],
['second provider', () => new FakeProvider({ id: 'beta', languages: ['auto', 'zh'], text: 'hello from beta' }), 'hello from beta'],
] as const)('VoiceService lifecycle (%s)', (_label, make, expectedText) => {
test('prepare runs in the background, reports progress, then transcribes', async () => {
const provider = make()
const { service } = createService([provider])
const id = provider.info.id
expect((await service.status(id)).preparation.phase).toBe('unprepared')
await expectRejects(service.transcribe(id, makeWav(1), 'auto', signal()), VoiceServiceError)
// prepare() must return while the provider is still downloading.
const started = await service.prepare(id)
expect(started.preparation.phase).toBe('downloading')
await provider.started
provider.report({ phase: 'downloading', step: 'model', completedBytes: 10, totalBytes: 100 })
expect((await service.status(id)).preparation).toMatchObject({ step: 'model', completedBytes: 10 })
provider.finishPrepare()
await flush()
expect((await service.status(id)).preparation.phase).toBe('ready')
const transcript = await service.transcribe(id, makeWav(2), 'auto', signal())
expect(transcript).toEqual({ text: expectedText, audioSeconds: 2, inferenceSeconds: 0.25 })
const removed = await service.removeAssets(id)
expect(removed.preparation.phase).toBe('unprepared')
expect(provider.removeCalls).toBe(1)
})
})
describe('VoiceService prepare', () => {
test('concurrent prepare calls join a single provider task', async () => {
const provider = new FakeProvider({ id: 'alpha' })
const { service } = createService([provider])
const [first, second] = await Promise.all([service.prepare('alpha'), service.prepare('alpha')])
await provider.started
await service.prepare('alpha')
expect(first.preparation.phase).toBe('downloading')
expect(second.preparation.phase).toBe('downloading')
expect(provider.prepareCalls).toBe(1)
provider.finishPrepare()
await flush()
})
test('does not start a download for assets that are already installed', async () => {
const provider = new FakeProvider({ id: 'alpha' })
provider.installed = true
const { service } = createService([provider])
await service.prepare('alpha')
await flush()
expect(provider.prepareCalls).toBe(0)
expect((await service.status('alpha')).preparation.phase).toBe('ready')
})
test('keeps the provider-reported unsupported-platform failure after prepare', async () => {
const provider = new FakeProvider({ id: 'alpha', unsupported: true })
const { service } = createService([provider])
await service.prepare('alpha')
await flush()
expect((await service.status('alpha')).preparation).toMatchObject({
phase: 'failed',
error: { reason: 'unsupported-platform' },
})
})
test('cancel aborts the provider, reports cancelled and allows a fresh prepare', async () => {
const provider = new FakeProvider({ id: 'alpha' })
const { service } = createService([provider])
await service.prepare('alpha')
await provider.started
provider.resetStarted()
const cancelled = await service.cancel('alpha')
expect(cancelled.preparation.phase).toBe('cancelled')
await flush()
expect((await service.status('alpha')).preparation.phase).toBe('cancelled')
const retry = await service.prepare('alpha')
expect(retry.preparation.phase).toBe('downloading')
await provider.started
expect(provider.prepareCalls).toBe(2)
provider.finishPrepare()
await flush()
expect((await service.status('alpha')).preparation.phase).toBe('ready')
})
test('a prepare issued right after cancel waits for the cancelled download to stop', async () => {
const provider = new FakeProvider({ id: 'alpha' })
provider.holdAbort = true
const { service } = createService([provider])
await service.prepare('alpha')
await provider.started
provider.resetStarted()
await service.cancel('alpha')
const retry = await service.prepare('alpha')
await flush()
// The cancelled download is still winding down, so no second one may start.
expect(retry.preparation.phase).toBe('downloading')
expect(provider.prepareCalls).toBe(1)
provider.releaseAbort()
await provider.started
expect(provider.prepareCalls).toBe(2)
provider.finishPrepare()
await flush()
expect((await service.status('alpha')).preparation.phase).toBe('ready')
})
test('cancel without a running download is a no-op status read', async () => {
const provider = new FakeProvider({ id: 'alpha' })
const { service } = createService([provider])
expect((await service.cancel('alpha')).preparation.phase).toBe('unprepared')
})
test('a failed download is reported with its reason and cleared by the next prepare', async () => {
const provider = new FakeProvider({ id: 'alpha' })
const { service } = createService([provider])
await service.prepare('alpha')
await provider.started
provider.resetStarted()
provider.report({
phase: 'failed',
error: { reason: 'dns', source: 'https://huggingface.co', message: 'getaddrinfo ENOTFOUND' },
})
provider.failPrepare(new Error('network down'))
await flush()
expect((await service.status('alpha')).preparation).toEqual({
phase: 'failed',
error: { reason: 'dns', source: 'https://huggingface.co', message: 'getaddrinfo ENOTFOUND' },
})
await service.prepare('alpha')
await provider.started
expect((await service.status('alpha')).preparation.phase).toBe('downloading')
provider.finishPrepare()
await flush()
})
test('an unreported provider error becomes an unknown failure', async () => {
const provider = new FakeProvider({ id: 'alpha' })
const { service } = createService([provider])
await service.prepare('alpha')
await provider.started
provider.failPrepare(new Error('disk exploded'))
await flush()
expect((await service.status('alpha')).preparation).toEqual({
phase: 'failed',
error: { reason: 'unknown', message: 'disk exploded' },
})
})
test('removeAssets cancels a running download before deleting', async () => {
const provider = new FakeProvider({ id: 'alpha' })
const { service } = createService([provider])
await service.prepare('alpha')
await provider.started
const removed = await service.removeAssets('alpha')
expect(removed.preparation.phase).toBe('unprepared')
expect(provider.removeCalls).toBe(1)
})
test('a prepare issued while assets are being removed waits for the removal to finish', async () => {
const provider = new FakeProvider({ id: 'alpha' })
provider.installed = true
let finishRemoval!: () => void
const removal = new Promise<void>(resolve => { finishRemoval = resolve })
const originalRemove = provider.preparation!.remove
provider.preparation!.remove = async () => {
await removal
await originalRemove()
}
const { service } = createService([provider])
const removing = service.removeAssets('alpha')
await flush()
await service.prepare('alpha')
await flush()
// The new download must not start (and get deleted) while removal is in flight.
expect(provider.prepareCalls).toBe(0)
finishRemoval()
await removing
await provider.started
expect(provider.removeCalls).toBe(1)
expect(provider.prepareCalls).toBe(1)
})
test('unknown providers are rejected by every provider operation', async () => {
const { service } = createService([new FakeProvider({ id: 'alpha' })])
for (const call of [
() => service.status('nope'),
() => service.prepare('nope'),
() => service.cancel('nope'),
() => service.removeAssets('nope'),
]) {
const error = await expectRejects(call(), VoiceServiceError)
expect(error.code).toBe('voice/unknown-provider')
expect(error.status).toBe(404)
}
})
})
describe('VoiceService transcribe', () => {
function readyService(overrides: ConstructorParameters<typeof FakeProvider>[0] = { id: 'alpha' }) {
const provider = new FakeProvider(overrides)
provider.installed = true
return { provider, ...createService([provider]) }
}
test('uses preferred provider and language when none are given', async () => {
const { service, provider, preferences } = readyService()
preferences.current = { ...preferences.current, providerId: 'alpha', language: 'zh' }
await service.transcribe(undefined, makeWav(1), undefined, signal())
expect(provider.transcribeCalls).toEqual([{ bytes: makeWav(1).byteLength, language: 'zh' }])
})
test('falls back to auto when the saved language is not offered by the provider', async () => {
const { service, provider, preferences } = readyService({ id: 'alpha', languages: ['auto', 'en'] })
preferences.current = { ...preferences.current, language: 'yue' }
await service.transcribe(undefined, makeWav(1), undefined, signal())
expect(provider.transcribeCalls[0]!.language).toBe('auto')
})
test('rejects an explicit language the provider does not list', async () => {
const { service } = readyService({ id: 'alpha', languages: ['auto', 'en'] })
const error = await expectRejects(service.transcribe('alpha', makeWav(1), 'ja', signal()), ApiError)
expect(error.statusCode).toBe(400)
})
test('rejects invalid audio before touching the provider', async () => {
const { service, provider } = readyService()
const error = await expectRejects(service.transcribe('alpha', new Uint8Array(100), 'auto', signal()), VoiceServiceError)
expect(error.code).toBe('voice/invalid-audio')
expect(provider.transcribeCalls).toHaveLength(0)
})
test('reports not-ready while assets are missing or still downloading', async () => {
const provider = new FakeProvider({ id: 'alpha' })
const { service } = createService([provider])
const missing = await expectRejects(service.transcribe('alpha', makeWav(1), 'auto', signal()), VoiceServiceError)
expect(missing.code).toBe('voice/not-ready')
expect(missing.status).toBe(409)
await service.prepare('alpha')
await provider.started
const downloading = await expectRejects(service.transcribe('alpha', makeWav(1), 'auto', signal()), VoiceServiceError)
expect(downloading.code).toBe('voice/not-ready')
expect(downloading.message).toContain('downloading')
expect(provider.transcribeCalls).toHaveLength(0)
provider.finishPrepare()
await flush()
})
test('still transcribes while the feature is disabled so settings can test it', async () => {
const { service, provider, preferences } = readyService()
expect(preferences.current.enabled).toBe(false)
await service.transcribe('alpha', makeWav(1), 'auto', signal())
expect(provider.transcribeCalls).toHaveLength(1)
})
test('wraps provider errors as voice/failed', async () => {
const { service, provider } = readyService()
provider.transcribeError = new Error('worker crashed')
const error = await expectRejects(service.transcribe('alpha', makeWav(1), 'auto', signal()), VoiceServiceError)
expect(error.code).toBe('voice/failed')
expect(error.status).toBe(500)
expect(error.message).toContain('worker crashed')
})
test('keeps a provider VoiceServiceError code instead of rewriting it to voice/failed', async () => {
const { service, provider } = readyService()
// A worker rejecting the audio is the caller's problem (400), and a model
// removed after the readiness check is a conflict (409); neither is a 500.
provider.transcribeError = new VoiceServiceError('voice/invalid-audio', 'Audio is not a WAV file')
const invalid = await expectRejects(service.transcribe('alpha', makeWav(1), 'auto', signal()), VoiceServiceError)
expect(invalid.code).toBe('voice/invalid-audio')
expect(invalid.status).toBe(400)
expect(invalid.message).toBe('Audio is not a WAV file')
provider.transcribeError = new VoiceServiceError('voice/not-ready', 'Speech model is not downloaded yet')
const notReady = await expectRejects(service.transcribe('alpha', makeWav(1), 'auto', signal()), VoiceServiceError)
expect(notReady.code).toBe('voice/not-ready')
expect(notReady.status).toBe(409)
})
test('rethrows the provider error untouched when the caller aborted', async () => {
const { service, provider } = readyService()
const controller = new AbortController()
const abortError = new DOMException('The operation was aborted', 'AbortError')
provider.transcribeError = abortError
controller.abort()
const error = await expectRejects(service.transcribe('alpha', makeWav(1), 'auto', controller.signal), DOMException)
expect(error).toBe(abortError)
})
test('runs the same transcribe path for two different providers', async () => {
const alpha = new FakeProvider({ id: 'alpha', text: 'A' })
const beta = new FakeProvider({ id: 'beta', text: 'B', needsAssets: false, location: 'cloud' })
alpha.installed = true
const { service } = createService([alpha, beta])
expect((await service.transcribe('alpha', makeWav(1), 'auto', signal())).text).toBe('A')
expect((await service.transcribe('beta', makeWav(1), 'auto', signal())).text).toBe('B')
})
})
describe('VoiceService preferences', () => {
test('applies partial updates without dropping other fields', async () => {
const { service, preferences } = createService([new FakeProvider({ id: 'alpha' })], {
preferences: { enabled: true, language: 'en' },
})
expect(await service.updatePreferences({ language: 'zh' })).toEqual({
enabled: true,
providerId: 'alpha',
language: 'zh',
})
expect(preferences.current.enabled).toBe(true)
})
test('rejects unknown providers, bad types and unsupported languages', async () => {
const { service, preferences } = createService([new FakeProvider({ id: 'alpha', languages: ['auto', 'en'] })], {
preferences: { providerId: 'alpha' },
})
const before = { ...preferences.current }
expect((await expectRejects(service.updatePreferences({ providerId: 'nope' }), VoiceServiceError)).code)
.toBe('voice/unknown-provider')
await expectRejects(service.updatePreferences({ enabled: 'yes' }), ApiError)
await expectRejects(service.updatePreferences({ language: 'fr' }), ApiError)
await expectRejects(service.updatePreferences({ language: 'ja' }), ApiError)
await expectRejects(service.updatePreferences(null), ApiError)
await expectRejects(service.updatePreferences([]), ApiError)
expect(preferences.current).toEqual(before)
})
test('switching provider resets a language the new provider cannot serve', async () => {
const { service } = createService(
[
new FakeProvider({ id: 'alpha', languages: ['auto', 'zh', 'yue'] }),
new FakeProvider({ id: 'beta', languages: ['auto', 'en'] }),
],
{ preferences: { providerId: 'alpha', language: 'yue' } },
)
expect(await service.updatePreferences({ providerId: 'beta' })).toMatchObject({
providerId: 'beta',
language: 'auto',
})
})
})
@@ -0,0 +1,83 @@
import { describe, expect, test } from 'bun:test'
import { VoiceServiceError } from '../errors.js'
import { VOICE_LIMITS } from '../types.js'
import { validateVoiceWav } from '../wav.js'
import { makeWav } from './fakeProvider.js'
function expectInvalid(bytes: Uint8Array, pattern: RegExp) {
try {
validateVoiceWav(bytes)
} catch (error) {
expect(error).toBeInstanceOf(VoiceServiceError)
expect((error as VoiceServiceError).code).toBe('voice/invalid-audio')
expect((error as VoiceServiceError).status).toBe(400)
expect((error as Error).message).toMatch(pattern)
return
}
throw new Error('expected validateVoiceWav to throw')
}
describe('validateVoiceWav', () => {
test('accepts a canonical 16 kHz mono PCM16 file and reports its duration', () => {
expect(validateVoiceWav(makeWav(1.5))).toEqual({ audioSeconds: 1.5 })
})
test('accepts audio exactly at the duration limit', () => {
expect(validateVoiceWav(makeWav(VOICE_LIMITS.maxAudioSeconds)).audioSeconds).toBe(VOICE_LIMITS.maxAudioSeconds)
})
test('accepts a view into a larger buffer (byteOffset is honoured)', () => {
const wav = makeWav(0.5)
const padded = new Uint8Array(wav.byteLength + 7)
padded.set(wav, 7)
expect(validateVoiceWav(padded.subarray(7)).audioSeconds).toBe(0.5)
})
test('rejects audio longer than the duration limit', () => {
expectInvalid(makeWav(VOICE_LIMITS.maxAudioSeconds + 1), /too long/)
})
test('rejects bodies larger than the byte limit before parsing them', () => {
expectInvalid(new Uint8Array(VOICE_LIMITS.maxAudioBytes + 1), /too large/)
})
test('rejects files shorter than a header and non-WAV data', () => {
expectInvalid(new Uint8Array(10), /not a WAV/)
expectInvalid(makeWav(0.1, { riff: 'RIFX' }), /not a WAV/)
expectInvalid(makeWav(0.1, { wave: 'AVI ' }), /not a WAV/)
})
test('rejects a header-only file as empty', () => {
expectInvalid(makeWav(0), /empty/)
})
test.each([
['stereo', { channels: 2 }, /mono/],
['44.1 kHz', { sampleRate: 44_100 }, /sample rate/],
['8-bit', { bits: 8 }, /bit depth/],
['float encoding', { format: 3 }, /PCM/],
] as const)('rejects %s audio', (_label, overrides, pattern) => {
expectInvalid(makeWav(0.1, overrides), pattern)
})
test('rejects a data length that disagrees with the file length', () => {
expectInvalid(makeWav(0.1, { dataLength: 100 }), /does not match/)
expectInvalid(makeWav(0.1, { dataLength: 0xffffffff }), /does not match/)
})
test('rejects a data chunk that is not a whole number of samples', () => {
const wav = makeWav(0.1)
const odd = wav.subarray(0, wav.byteLength - 1)
new DataView(odd.buffer, odd.byteOffset, odd.byteLength).setUint32(40, odd.byteLength - 44, true)
expectInvalid(odd, /whole number of samples/)
})
test('rejects an extended fmt chunk (non-canonical header)', () => {
const wav = makeWav(0.1)
new DataView(wav.buffer).setUint32(16, 18, true)
expectInvalid(wav, /canonical/)
})
})
@@ -0,0 +1,38 @@
import { createDesktopUiVoicePreferencesStore } from './preferencesStore.js'
import { VoiceProviderRegistry } from './registry.js'
import { createSenseVoiceProvider } from './sensevoice/index.js'
import type { SpeechProvider } from './types.js'
import { VoiceService, type VoiceServiceOptions } from './voiceService.js'
/**
* Providers shipped with the server. Adding a provider means adding one
* factory here; nothing else in the voice service or API changes.
*/
export function createDefaultProviders(): SpeechProvider[] {
return [createSenseVoiceProvider()]
}
export function createVoiceService(
providers: SpeechProvider[],
options: Partial<Omit<VoiceServiceOptions, 'registry'>> = {},
): VoiceService {
const registry = new VoiceProviderRegistry()
for (const provider of providers) registry.register(provider)
return new VoiceService({
registry,
preferences: options.preferences ?? createDesktopUiVoicePreferencesStore(),
})
}
let instance: VoiceService | undefined
/** Process-wide service used by the HTTP API. Providers are created on first use. */
export function getVoiceService(): VoiceService {
instance ??= createVoiceService(createDefaultProviders())
return instance
}
/** Test seam: drop the cached service so the next call rebuilds it. */
export function resetVoiceServiceForTests(): void {
instance = undefined
}
@@ -0,0 +1,378 @@
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'bun:test'
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { VoiceDownloadError, downloadAsset, partPathFor, type DownloadProgress } from './index.js'
import { writeFully } from './downloader.js'
import { assetFor, bypassProxyForLoopback, makeContent, startFixture, type Fixture } from './httpFixture.testUtil.js'
const content = makeContent(300_000)
const noSleep = async () => {}
let dir: string
let fixtures: Fixture[]
let restoreProxyEnv: () => void
beforeAll(() => { restoreProxyEnv = bypassProxyForLoopback() })
afterAll(() => { restoreProxyEnv() })
async function serve(behavior = {}, body = content): Promise<Fixture> {
const fixture = await startFixture(body, behavior)
fixtures.push(fixture)
return fixture
}
async function exists(path: string): Promise<boolean> {
return stat(path).then(() => true, () => false)
}
beforeEach(async () => {
dir = await mkdtemp(join(tmpdir(), 'voice-download-'))
fixtures = []
})
afterEach(async () => {
await Promise.all(fixtures.map(fixture => fixture.close()))
await rm(dir, { recursive: true, force: true })
})
describe('downloadAsset', () => {
it('downloads a pinned file, verifies it and reports progress up to the total', async () => {
const server = await serve()
const destination = join(dir, 'out', 'file.bin')
const events: DownloadProgress[] = []
const result = await downloadAsset(assetFor(content, [`${server.origin}/file`]), destination, {
onProgress: event => events.push(event),
progressIntervalMs: 0,
})
expect(Buffer.compare(await readFile(destination), content)).toBe(0)
expect(await exists(partPathFor(destination))).toBe(false)
expect(result).toEqual({ source: server.origin })
expect(events.at(-1)).toEqual({ completedBytes: content.length, totalBytes: content.length, source: server.origin })
const completed = events.map(event => event.completedBytes)
expect([...completed].sort((a, b) => a - b)).toEqual(completed)
expect(server.gets()[0]?.range).toBeUndefined()
})
it('throttles progress callbacks to the configured interval', async () => {
const server = await serve({ chunkDelayMs: 1 })
const events: DownloadProgress[] = []
await downloadAsset(assetFor(content, [`${server.origin}/file`]), join(dir, 'file.bin'), {
onProgress: event => events.push(event),
progressIntervalMs: 60_000,
})
// First event on connect, final event on completion; nothing in between.
expect(events.map(event => event.completedBytes)).toEqual([0, content.length])
})
it('resumes with a Range request after the connection drops mid-download', async () => {
const server = await serve({ breakAfter: 100_000, breakRequests: 1, chunkDelayMs: 2 })
const destination = join(dir, 'file.bin')
const events: DownloadProgress[] = []
const sleeps: number[] = []
const result = await downloadAsset(assetFor(content, [`${server.origin}/file`]), destination, {
onProgress: event => events.push(event),
progressIntervalMs: 0,
sleep: async ms => { sleeps.push(ms) },
})
expect(Buffer.compare(await readFile(destination), content)).toBe(0)
const gets = server.gets()
expect(gets).toHaveLength(2)
expect(gets[0]?.range).toBeUndefined()
const resumeOffset = Number(/^bytes=(\d+)-$/.exec(gets[1]?.range ?? '')?.[1])
expect(resumeOffset).toBeGreaterThan(0)
expect(resumeOffset).toBeLessThan(content.length)
expect(result.resumedFromBytes).toBe(resumeOffset)
expect(events.at(-1)?.resumedFromBytes).toBe(resumeOffset)
expect(sleeps).toHaveLength(1)
})
it('drops the last chunk of an interrupted transfer so stray trailing bytes never enter the resumed prefix', async () => {
const server = await serve()
const destination = join(dir, 'file.bin')
let calls = 0
const flaky = (input: string, init?: RequestInit & Record<string, unknown>): Promise<Response> => {
if (calls++ > 0) return fetch(input, init as RequestInit)
// Real bytes, then a small junk chunk, then a reset: the shape of the Bun client glitch.
const body = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(content.subarray(0, 50_000))
controller.enqueue(new TextEncoder().encode('Connection: close\r\n\r\n'))
},
pull(controller) { controller.error(new Error('The socket connection was closed unexpectedly')) },
})
return Promise.resolve(new Response(body, { status: 200, headers: { 'content-length': String(content.length) } }))
}
const result = await downloadAsset(assetFor(content, [`${server.origin}/file`]), destination, { fetch: flaky, sleep: noSleep })
expect(Buffer.compare(await readFile(destination), content)).toBe(0)
expect(server.gets()).toHaveLength(1)
expect(server.gets()[0]?.range).toBe('bytes=50000-')
expect(result.resumedFromBytes).toBe(50_000)
})
it('backs off exponentially between retries', async () => {
const server = await serve({ breakAfter: 20_000, breakRequests: 3 })
const sleeps: number[] = []
await downloadAsset(assetFor(content, [`${server.origin}/file`]), join(dir, 'file.bin'), {
backoffMs: 100,
sleep: async ms => { sleeps.push(ms) },
})
expect(sleeps).toEqual([100, 200, 400])
})
it('resumes from a .part left by an earlier run, feeding the prefix into the digest', async () => {
const server = await serve()
const destination = join(dir, 'file.bin')
await writeFile(partPathFor(destination), content.subarray(0, 123_456))
const result = await downloadAsset(assetFor(content, [`${server.origin}/file`]), destination)
expect(Buffer.compare(await readFile(destination), content)).toBe(0)
expect(server.gets()).toHaveLength(1)
expect(server.gets()[0]?.range).toBe('bytes=123456-')
expect(result.resumedFromBytes).toBe(123_456)
})
it('restarts from zero when the server ignores Range and answers 200', async () => {
const server = await serve({ ignoreRange: true })
const destination = join(dir, 'file.bin')
await writeFile(partPathFor(destination), content.subarray(0, 50_000))
const result = await downloadAsset(assetFor(content, [`${server.origin}/file`]), destination)
expect(Buffer.compare(await readFile(destination), content)).toBe(0)
expect(server.gets()).toHaveLength(1)
expect(result.resumedFromBytes).toBeUndefined()
})
it('discards the .part and starts over on 416', async () => {
const server = await serve({ rangeNotSatisfiable: true })
const destination = join(dir, 'file.bin')
await writeFile(partPathFor(destination), content.subarray(0, 50_000))
await downloadAsset(assetFor(content, [`${server.origin}/file`]), destination)
expect(Buffer.compare(await readFile(destination), content)).toBe(0)
expect(server.gets().map(request => request.range)).toEqual(['bytes=50000-', undefined])
})
it('drops a corrupt resumed prefix and refetches from zero', async () => {
const server = await serve()
const destination = join(dir, 'file.bin')
await writeFile(partPathFor(destination), Buffer.alloc(40_000, 1))
await downloadAsset(assetFor(content, [`${server.origin}/file`]), destination)
expect(Buffer.compare(await readFile(destination), content)).toBe(0)
expect(server.gets().map(request => request.range)).toEqual(['bytes=40000-', undefined])
})
it('deletes the .part and fails with integrity when a fresh download does not match', async () => {
const tampered = Buffer.from(content)
tampered[1000] ^= 0xff
const server = await serve({}, tampered)
const destination = join(dir, 'file.bin')
const error = await downloadAsset(assetFor(content, [`${server.origin}/file`]), destination).catch(e => e)
expect(error).toBeInstanceOf(VoiceDownloadError)
expect(error.failure.reason).toBe('integrity')
expect(await exists(partPathFor(destination))).toBe(false)
expect(await exists(destination)).toBe(false)
})
it('verifies a complete .part without touching the network', async () => {
const server = await serve()
const destination = join(dir, 'file.bin')
await writeFile(partPathFor(destination), content)
await downloadAsset(assetFor(content, [`${server.origin}/file`]), destination)
expect(Buffer.compare(await readFile(destination), content)).toBe(0)
expect(server.gets()).toHaveLength(0)
})
it('keeps the .part when cancelled and continues from it next time', async () => {
const server = await serve({ chunkDelayMs: 3 })
const destination = join(dir, 'file.bin')
const asset = assetFor(content, [`${server.origin}/file`])
const controller = new AbortController()
const cancelled = await downloadAsset(asset, destination, {
signal: controller.signal,
progressIntervalMs: 0,
onProgress: event => { if (event.completedBytes >= 100_000) controller.abort() },
}).catch(error => error)
expect(controller.signal.aborted).toBe(true)
expect(cancelled).not.toBeInstanceOf(VoiceDownloadError)
const kept = (await stat(partPathFor(destination))).size
expect(kept).toBeGreaterThanOrEqual(100_000)
expect(await exists(destination)).toBe(false)
const result = await downloadAsset(asset, destination)
expect(Buffer.compare(await readFile(destination), content)).toBe(0)
expect(result.resumedFromBytes).toBe(kept)
expect(server.gets().at(-1)?.range).toBe(`bytes=${kept}-`)
})
it('gives up on a source after the retry budget and reports a network failure', async () => {
const server = await serve({ breakAfter: 40_000, chunkDelayMs: 2 })
const destination = join(dir, 'file.bin')
const error = await downloadAsset(assetFor(content, [`${server.origin}/file`]), destination, {
maxRetries: 2,
sleep: noSleep,
}).catch(e => e)
expect(error).toBeInstanceOf(VoiceDownloadError)
expect(error.failure.reason).toBe('network')
expect(error.failure.source).toBe(server.origin)
expect(server.gets()).toHaveLength(3)
// Progress is kept for the next prepare.
expect((await stat(partPathFor(destination))).size).toBeGreaterThan(0)
})
it('falls back to the next source only after retries on the first are exhausted', async () => {
const flaky = await serve({ breakAfter: 10_000 })
const good = await serve()
const destination = join(dir, 'file.bin')
const result = await downloadAsset(
assetFor(content, [`${flaky.origin}/file`, `${good.origin}/file`]),
destination,
{ maxRetries: 1, sleep: noSleep, probeTimeoutMs: 1000 },
)
expect(result.source).toBe(good.origin)
expect(Buffer.compare(await readFile(destination), content)).toBe(0)
expect(flaky.gets()).toHaveLength(2)
expect(good.gets().length).toBeGreaterThanOrEqual(1)
})
it('falls back on HTTP errors immediately', async () => {
const broken = await serve({ status: 503 })
const good = await serve()
const result = await downloadAsset(
assetFor(content, [`${broken.origin}/file`, `${good.origin}/file`]),
join(dir, 'file.bin'),
{ sleep: noSleep },
)
expect(result.source).toBe(good.origin)
expect(broken.gets()).toHaveLength(1)
})
it('reports the HTTP status when every source fails', async () => {
const broken = await serve({ status: 404 })
const error = await downloadAsset(assetFor(content, [`${broken.origin}/file`]), join(dir, 'file.bin'), {
sleep: noSleep,
}).catch(e => e)
expect(error.failure).toMatchObject({ reason: 'http', status: 404, source: broken.origin })
})
it('tries the fastest mirror first according to a HEAD probe', async () => {
const slow = await serve({ headDelayMs: 500 })
const fast = await serve()
const result = await downloadAsset(
assetFor(content, [`${slow.origin}/file`, `${fast.origin}/file`]),
join(dir, 'file.bin'),
{ probeTimeoutMs: 3000 },
)
expect(result.source).toBe(fast.origin)
expect(slow.gets()).toHaveLength(0)
})
it('does not switch source on a storage failure', async () => {
const first = await serve()
const second = await serve()
const blocker = join(dir, 'blocker')
await writeFile(blocker, 'a file where a directory is needed')
const error = await downloadAsset(
assetFor(content, [`${first.origin}/file`, `${second.origin}/file`]),
join(blocker, 'nested', 'file.bin'),
).catch(e => e)
expect(error.failure.reason).toBe('storage')
expect(first.gets()).toHaveLength(0)
expect(second.gets()).toHaveLength(0)
})
it('classifies a stalled transfer as a timeout and keeps what arrived', async () => {
const server = await serve({ stallAfter: 40_000, chunkDelayMs: 2 })
const destination = join(dir, 'file.bin')
const error = await downloadAsset(assetFor(content, [`${server.origin}/file`]), destination, {
idleTimeoutMs: 150,
maxRetries: 0,
}).catch(e => e)
expect(error.failure.reason).toBe('timeout')
expect((await stat(partPathFor(destination))).size).toBeGreaterThan(0)
})
it('classifies certificate and DNS failures and never leaks URL credentials', async () => {
const asset = assetFor(content, ['https://user:secret@files.example.test/a/file.bin?token=abc'])
const failWith = (message: string, code?: string) => async () => {
throw Object.assign(new Error(message), code ? { code } : {})
}
const cert = await downloadAsset(asset, join(dir, 'a.bin'), {
fetch: failWith('unable to verify the first certificate', 'UNABLE_TO_VERIFY_LEAF_SIGNATURE'),
}).catch(e => e)
expect(cert.failure.reason).toBe('certificate')
const dns = await downloadAsset(asset, join(dir, 'b.bin'), {
fetch: failWith('getaddrinfo ENOTFOUND files.example.test https://user:secret@files.example.test/a?token=abc', 'ENOTFOUND'),
}).catch(e => e)
expect(dns.failure.reason).toBe('dns')
for (const error of [cert, dns]) {
const serialized = JSON.stringify(error.failure)
expect(serialized).not.toContain('secret')
expect(serialized).not.toContain('token=abc')
expect(error.failure.source).toBe('https://files.example.test')
}
})
it('creates the destination directory and applies fetch options such as a proxy', async () => {
const server = await serve()
const seen: unknown[] = []
await downloadAsset(assetFor(content, [`${server.origin}/file`]), join(dir, 'a', 'b', 'file.bin'), {
fetchOptions: () => ({ proxy: 'http://proxy.invalid:1' }),
fetch: async (input, init) => {
seen.push((init as { proxy?: string }).proxy)
const { proxy: _proxy, ...rest } = init as Record<string, unknown>
return fetch(input, rest as RequestInit)
},
})
expect(seen).toEqual(['http://proxy.invalid:1'])
await mkdir(join(dir, 'a', 'b'), { recursive: true })
})
})
describe('writeFully', () => {
it('keeps writing until a handle that accepts only part of each call has taken every byte', async () => {
const stored: number[] = []
const handle = {
async write(buffer: Uint8Array, offset: number, length: number) {
const accepted = Math.min(3, length)
stored.push(...buffer.subarray(offset, offset + accepted))
return { bytesWritten: accepted }
},
}
await writeFully(handle, Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]))
expect(stored).toEqual([1, 2, 3, 4, 5, 6, 7, 8])
})
it('fails instead of looping when the file system makes no progress', async () => {
const handle = { write: async () => ({ bytesWritten: 0 }) }
await expect(writeFully(handle, Uint8Array.from([1, 2, 3]))).rejects.toThrow('Short write')
})
})
@@ -0,0 +1,441 @@
import { createHash, type Hash } from 'node:crypto'
import { createReadStream } from 'node:fs'
import { mkdir, open, rename, rm, stat } from 'node:fs/promises'
import { dirname } from 'node:path'
import {
VoiceDownloadError,
isTerminal,
isTransient,
sanitizeOrigin,
scrubUrls,
toDownloadError,
} from './failure.js'
export type HashAlgorithm = 'sha256' | 'sha512'
export interface DownloadHash {
algorithm: HashAlgorithm
/** `hex` for sha256 files, `base64` for npm `sha512-...` integrity values. */
encoding: 'hex' | 'base64'
value: string
}
/** A pinned file: fixed size, fixed digest, one or more interchangeable URLs. */
export interface DownloadAsset {
/** Resource label shown in progress and failures, e.g. `model.int8.onnx`. */
name: string
bytes: number
hash: DownloadHash
/** Mirrors in preference order; the fastest responder is tried first. */
urls: string[]
}
export interface DownloadProgress {
completedBytes: number
totalBytes: number
/** Bytes already on disk when this transfer resumed; absent for a fresh start. */
resumedFromBytes?: number
/** Origin serving the transfer. */
source: string
}
export type FetchLike = (input: string, init?: RequestInit & Record<string, unknown>) => Promise<Response>
export interface DownloadOptions {
fetch?: FetchLike
/** Extra per-request fetch options such as the configured network proxy. */
fetchOptions?: (url: string) => Promise<Record<string, unknown>> | Record<string, unknown>
signal?: AbortSignal
onProgress?: (progress: DownloadProgress) => void
/** Retries per source after network/timeout interruptions. Default 5. */
maxRetries?: number
/** First backoff delay; doubles each retry up to `maxBackoffMs`. Default 500. */
backoffMs?: number
maxBackoffMs?: number
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>
/** Race HEAD probes across mirrors, at most this long. Default 3000. */
probeTimeoutMs?: number
/** Abort a transfer that receives nothing for this long. Default 30000. */
idleTimeoutMs?: number
/** Minimum gap between progress callbacks. Default 100. */
progressIntervalMs?: number
}
export interface DownloadResult {
/** Origin that delivered the final bytes. */
source: string
/** Last non-zero offset the transfer resumed from, if any resume happened. */
resumedFromBytes?: number
}
export const DEFAULT_MAX_RETRIES = 5
export function partPathFor(destination: string): string {
return `${destination}.part`
}
export function defaultSleep(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
if (signal?.aborted) {
reject(signal.reason)
return
}
const timer = setTimeout(() => {
signal?.removeEventListener('abort', onAbort)
resolve()
}, ms)
const onAbort = () => {
clearTimeout(timer)
reject(signal!.reason)
}
signal?.addEventListener('abort', onAbort, { once: true })
})
}
function storageError(error: unknown, resource: string): VoiceDownloadError {
if (error instanceof VoiceDownloadError) return error
const message = error instanceof Error ? error.message : String(error)
return new VoiceDownloadError(
{ reason: 'storage', resource, message: scrubUrls(message) },
{ cause: error },
)
}
async function fsCall<T>(resource: string, operation: () => Promise<T>): Promise<T> {
try {
return await operation()
} catch (error) {
throw storageError(error, resource)
}
}
/**
* Writes the whole chunk. The running hash covers the network bytes, so a short
* write that went unnoticed would leave a file that verifies yet is missing a
* span; loop until everything is on disk and fail loudly if no progress is made.
*/
export async function writeFully(
handle: { write(buffer: Uint8Array, offset: number, length: number): Promise<{ bytesWritten: number }> },
chunk: Uint8Array,
): Promise<void> {
let written = 0
while (written < chunk.byteLength) {
const { bytesWritten } = await handle.write(chunk, written, chunk.byteLength - written)
if (bytesWritten <= 0) throw new Error('Short write: the file system accepted no more data')
written += bytesWritten
}
}
async function partSize(path: string, resource: string): Promise<number> {
try {
return (await stat(path)).size
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return 0
throw storageError(error, resource)
}
}
async function removeQuietly(path: string): Promise<void> {
await rm(path, { force: true }).catch(() => {})
}
function feedPrefix(path: string, length: number, hash: Hash): Promise<void> {
return new Promise((resolve, reject) => {
const stream = createReadStream(path, { start: 0, end: length - 1 })
stream.on('data', chunk => hash.update(chunk))
stream.once('error', reject)
stream.once('end', () => resolve())
})
}
function digestMatches(hash: Hash, expected: DownloadHash): boolean {
const actual = hash.digest(expected.encoding)
return expected.encoding === 'hex' ? actual === expected.value.toLowerCase() : actual === expected.value
}
function parseContentRange(value: string | null): { start: number; end: number; total: number | undefined } | undefined {
const match = /^bytes (\d+)-(\d+)\/(\d+|\*)$/.exec(value?.trim() ?? '')
if (!match) return undefined
return { start: Number(match[1]), end: Number(match[2]), total: match[3] === '*' ? undefined : Number(match[3]) }
}
/** Races cheap HEAD probes; unreachable mirrors are kept as later fallbacks. */
async function orderSources(
urls: string[],
fetchImpl: FetchLike,
resolveOptions: (url: string) => Promise<Record<string, unknown>>,
timeoutMs: number,
signal: AbortSignal | undefined,
): Promise<string[]> {
if (urls.length < 2) return urls
const stop = new AbortController()
const timer = setTimeout(() => stop.abort(new Error('probe timeout')), timeoutMs)
const combined = signal ? AbortSignal.any([signal, stop.signal]) : stop.signal
try {
const winner = await Promise.any(urls.map(async url => {
const response = await fetchImpl(url, {
method: 'HEAD',
redirect: 'follow',
// Bun re-issues a request on a reused socket that resets mid-body and
// splices the retry into the stream; a fresh connection avoids that.
keepalive: false,
signal: combined,
...(await resolveOptions(url)),
})
await response.body?.cancel().catch(() => {})
if (response.status >= 400) throw new Error(`probe ${response.status}`)
return url
}))
return [winner, ...urls.filter(url => url !== winner)]
} catch {
signal?.throwIfAborted()
return urls
} finally {
clearTimeout(timer)
stop.abort(new Error('probe finished'))
}
}
/**
* Downloads one pinned file with resume, retry and mirror fallback.
*
* The partial file is `<destination>.part` (a deterministic name), so an
* interrupted, cancelled or crashed download continues from its byte offset the
* next time this is called. Only a digest mismatch deletes it. The final file
* appears atomically via rename, and only after the digest matches.
*/
export async function downloadAsset(
asset: DownloadAsset,
destination: string,
options: DownloadOptions = {},
): Promise<DownloadResult> {
const fetchImpl: FetchLike = options.fetch ?? ((input, init) => fetch(input, init as RequestInit))
const resolveOptions = async (url: string): Promise<Record<string, unknown>> =>
(await options.fetchOptions?.(url)) ?? {}
const sleep = options.sleep ?? defaultSleep
const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES
const backoffMs = options.backoffMs ?? 500
const maxBackoffMs = options.maxBackoffMs ?? 10_000
const idleTimeoutMs = options.idleTimeoutMs ?? 30_000
const progressIntervalMs = options.progressIntervalMs ?? 100
const part = partPathFor(destination)
const { signal } = options
const resource = asset.name
let resumedFromBytes: number | undefined
let lastProgressAt = 0
const report = (completedBytes: number, source: string, force: boolean): void => {
if (!options.onProgress) return
const now = performance.now()
if (!force && now - lastProgressAt < progressIntervalMs) return
lastProgressAt = now
options.onProgress({
completedBytes,
totalBytes: asset.bytes,
source,
...(resumedFromBytes ? { resumedFromBytes } : {}),
})
}
await fsCall(resource, () => mkdir(dirname(destination), { recursive: true }))
/** One transfer against one URL. Returns `restart` when the partial file was discarded. */
const transfer = async (url: string, allowResume: boolean): Promise<DownloadResult | 'restart'> => {
const source = sanitizeOrigin(url)
let offset = allowResume ? await partSize(part, resource) : 0
if (!allowResume) await removeQuietly(part)
if (offset > asset.bytes) {
await removeQuietly(part)
offset = 0
}
const hash = createHash(asset.hash.algorithm)
// A complete .part from an earlier run only needs verification.
if (offset === asset.bytes) {
await fsCall(resource, () => feedPrefix(part, offset, hash))
if (!digestMatches(hash, asset.hash)) {
await removeQuietly(part)
return 'restart'
}
await fsCall(resource, () => rename(part, destination))
resumedFromBytes = offset
report(asset.bytes, source, true)
return { source, resumedFromBytes: offset }
}
const attemptAbort = new AbortController()
let idleTimer: ReturnType<typeof setTimeout> | undefined
const armIdle = (): void => {
clearTimeout(idleTimer)
idleTimer = setTimeout(() => {
attemptAbort.abort(new DOMException('No data received', 'TimeoutError'))
}, idleTimeoutMs)
}
const combined = signal ? AbortSignal.any([signal, attemptAbort.signal]) : attemptAbort.signal
let reader: ReadableStreamDefaultReader<Uint8Array> | undefined
let handle: Awaited<ReturnType<typeof open>> | undefined
try {
armIdle()
let response: Response
try {
response = await fetchImpl(url, {
redirect: 'follow',
keepalive: false,
signal: combined,
headers: {
'Accept-Encoding': 'identity',
...(offset > 0 ? { Range: `bytes=${offset}-` } : {}),
},
...(await resolveOptions(url)),
})
} catch (error) {
signal?.throwIfAborted()
if (attemptAbort.signal.aborted) {
throw new VoiceDownloadError({ reason: 'timeout', source, resource, message: 'No data received before the timeout' })
}
throw toDownloadError(error, { source, resource })
}
if (response.status === 416) {
await response.body?.cancel().catch(() => {})
if (offset === 0) {
throw new VoiceDownloadError({ reason: 'http', source, resource, status: 416, message: 'HTTP 416 for a full download' })
}
await removeQuietly(part)
return 'restart'
}
let resumed = false
if (response.status === 206) {
const range = parseContentRange(response.headers.get('content-range'))
const consistent = offset > 0 && range
&& range.start === offset && range.end === asset.bytes - 1
&& (range.total === undefined || range.total === asset.bytes)
if (!consistent) {
await response.body?.cancel().catch(() => {})
await removeQuietly(part)
return 'restart'
}
resumed = true
} else if (response.status === 200) {
const length = response.headers.get('content-length')
if (length !== null && Number(length) !== asset.bytes) {
await response.body?.cancel().catch(() => {})
throw new VoiceDownloadError({
reason: 'integrity', source, resource,
message: `Server reported ${length} bytes, expected ${asset.bytes}`,
})
}
offset = 0
} else {
await response.body?.cancel().catch(() => {})
throw new VoiceDownloadError({
reason: 'http', source, resource, status: response.status,
message: `HTTP ${response.status} from ${source}`,
})
}
if (!response.body) {
throw new VoiceDownloadError({ reason: 'network', source, resource, message: 'Response had no body' })
}
if (resumed) {
await fsCall(resource, () => feedPrefix(part, offset, hash))
resumedFromBytes = offset
}
handle = await fsCall(resource, () => open(part, resumed ? 'a' : 'w'))
let completed = offset
report(completed, source, true)
reader = response.body.getReader()
let chunkStart = completed
try {
for (;;) {
const { done, value } = await reader.read()
if (done) break
armIdle()
if (completed + value.byteLength > asset.bytes) {
await removeQuietly(part)
throw new VoiceDownloadError({
reason: 'integrity', source, resource,
message: `Received more than the expected ${asset.bytes} bytes`,
})
}
hash.update(value)
chunkStart = completed
await fsCall(resource, () => writeFully(handle!, value))
completed += value.byteLength
report(completed, source, false)
}
} catch (error) {
signal?.throwIfAborted()
// Bun's fetch can append a few bytes of request text to the last chunk when the
// server resets mid-body. Dropping that chunk keeps the resumed prefix clean.
await handle.truncate(chunkStart).catch(() => {})
if (attemptAbort.signal.aborted) {
throw new VoiceDownloadError({ reason: 'timeout', source, resource, message: 'Transfer stalled' })
}
throw toDownloadError(error, { source, resource })
}
if (completed < asset.bytes) {
throw new VoiceDownloadError({
reason: 'network', source, resource,
message: `Connection closed after ${completed} of ${asset.bytes} bytes`,
})
}
await handle.close()
handle = undefined
if (!digestMatches(hash, asset.hash)) {
await removeQuietly(part)
// A bad prefix from an older run is recoverable; a bad fresh download is not.
if (resumed) return 'restart'
throw new VoiceDownloadError({
reason: 'integrity', source, resource,
message: `${asset.hash.algorithm} mismatch for ${asset.name}`,
})
}
await fsCall(resource, () => rename(part, destination))
report(asset.bytes, source, true)
return { source, ...(resumedFromBytes ? { resumedFromBytes } : {}) }
} finally {
clearTimeout(idleTimer)
attemptAbort.abort()
await reader?.cancel().catch(() => {})
await handle?.close().catch(() => {})
}
}
/** Retry loop for one URL: a discarded partial file restarts once from zero. */
const attempt = async (url: string): Promise<DownloadResult> => {
const first = await transfer(url, true)
if (first !== 'restart') return first
const second = await transfer(url, false)
if (second !== 'restart') return second
throw new VoiceDownloadError({
reason: 'integrity', source: sanitizeOrigin(url), resource,
message: `${asset.hash.algorithm} mismatch for ${asset.name}`,
})
}
const ordered = await orderSources(asset.urls, fetchImpl, resolveOptions, options.probeTimeoutMs ?? 3000, signal)
let lastError: VoiceDownloadError | undefined
for (const url of ordered) {
for (let retry = 0; ; retry++) {
try {
return await attempt(url)
} catch (error) {
signal?.throwIfAborted()
const failure = toDownloadError(error, { source: sanitizeOrigin(url), resource })
lastError = failure
const reason = failure.failure.reason
if (isTerminal(reason)) throw failure
if (!isTransient(reason) || retry >= maxRetries) break
await sleep(Math.min(backoffMs * 2 ** retry, maxBackoffMs), signal)
}
}
}
throw lastError ?? new VoiceDownloadError({ reason: 'unknown', resource, message: 'No download source configured' })
}
@@ -0,0 +1,104 @@
import type { VoiceFailure, VoiceFailureReason } from '../types.js'
/** A download that could not complete; `failure` is safe to show in the UI. */
export class VoiceDownloadError extends Error {
readonly failure: VoiceFailure
constructor(failure: VoiceFailure, options?: { cause?: unknown }) {
super(failure.message, options)
this.name = 'VoiceDownloadError'
this.failure = failure
}
}
/** Origin only: drops credentials, path and query so tokens never reach the UI or logs. */
export function sanitizeOrigin(url: string): string {
try {
return new URL(url).origin
} catch {
return 'invalid-url'
}
}
/** Replaces any URL inside a message with its credential-free origin + path. */
export function scrubUrls(message: string): string {
return message.replace(/https?:\/\/[^\s"')]+/g, raw => {
try {
const url = new URL(raw)
return `${url.origin}${url.pathname}`
} catch {
return 'url'
}
})
}
const STORAGE_CODES = new Set([
'ENOSPC', 'EDQUOT', 'EACCES', 'EPERM', 'EROFS', 'EMFILE', 'ENFILE', 'EIO', 'ENOENT', 'ENOTDIR', 'EISDIR', 'EBUSY', 'EEXIST',
])
const DNS_CODES = new Set(['ENOTFOUND', 'EAI_AGAIN', 'EAI_NODATA', 'EAI_FAIL', 'DNS_FAILURE'])
const TIMEOUT_CODES = new Set(['ETIMEDOUT', 'ESOCKETTIMEDOUT', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT', 'Timeout', 'TimeoutError'])
const CERTIFICATE_PATTERN =
/CERT_|CERTIFICATE|UNABLE_TO_VERIFY|SELF.SIGNED|ERR_TLS|SSL|UnknownIssuer|InvalidCertificate|UNABLE_TO_GET_ISSUER/i
function errorCodes(error: unknown): string[] {
const codes: string[] = []
let current: unknown = error
for (let depth = 0; depth < 4 && current instanceof Error; depth++) {
const code = (current as { code?: unknown }).code
if (typeof code === 'string') codes.push(code)
if (current.name) codes.push(current.name)
current = (current as { cause?: unknown }).cause
}
return codes
}
function messageChain(error: unknown): string {
const parts: string[] = []
let current: unknown = error
for (let depth = 0; depth < 4 && current instanceof Error; depth++) {
parts.push(current.message)
current = (current as { cause?: unknown }).cause
}
return parts.join(' | ')
}
/** Maps a thrown network/fs error to a stable reason. Unknown shapes stay `unknown`. */
export function classifyError(error: unknown): VoiceFailureReason {
if (error instanceof VoiceDownloadError) return error.failure.reason
const codes = errorCodes(error)
const text = messageChain(error)
if (codes.some(code => STORAGE_CODES.has(code))) return 'storage'
if (codes.some(code => DNS_CODES.has(code)) || /getaddrinfo|dns/i.test(text)) return 'dns'
if (codes.some(code => CERTIFICATE_PATTERN.test(code)) || CERTIFICATE_PATTERN.test(text)) return 'certificate'
if (codes.some(code => TIMEOUT_CODES.has(code)) || /timed? ?out/i.test(text)) return 'timeout'
if (
codes.some(code => /^(ECONN|EPIPE|ENET|EHOST|UND_ERR|ConnectionRefused|ConnectionClosed|ConnectionReset|FailedToOpenSocket)/i.test(code))
|| /fetch failed|socket|connection|network|closed|terminated|unable to connect|ECONN|EPIPE/i.test(text)
) {
return 'network'
}
return 'unknown'
}
/** Failures worth another attempt against the same source, resuming from `.part`. */
export function isTransient(reason: VoiceFailureReason): boolean {
return reason === 'network' || reason === 'timeout'
}
/** Failures another source cannot fix: local disk problems and programming errors. */
export function isTerminal(reason: VoiceFailureReason): boolean {
return reason === 'storage' || reason === 'unknown' || reason === 'unsupported-platform'
}
export function toDownloadError(
error: unknown,
context: { source: string; resource: string },
): VoiceDownloadError {
if (error instanceof VoiceDownloadError) return error
const reason = classifyError(error)
const raw = error instanceof Error ? error.message : String(error)
return new VoiceDownloadError(
{ reason, source: context.source, resource: context.resource, message: scrubUrls(raw) },
{ cause: error },
)
}
@@ -0,0 +1,161 @@
/** Loopback HTTP fixture shared by downloader and runtime tests. Test-only. */
import { createHash } from 'node:crypto'
import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'
import type { DownloadAsset } from './downloader.js'
export interface RecordedRequest {
method: string
range: string | undefined
path: string
}
export interface FixtureBehavior {
/** Ignore Range headers and always answer 200 with the full body. */
ignoreRange?: boolean
/** Answer 416 to any request that carries a Range header. */
rangeNotSatisfiable?: boolean
/** Force this status for GET requests (HEAD stays 200 unless `headStatus`). */
status?: number
headStatus?: number
headDelayMs?: number
/** Destroy the socket after sending this many body bytes, for the first N GETs. */
breakAfter?: number
breakRequests?: number
/** Pause between 16 KiB chunks. */
chunkDelayMs?: number
/** Stall forever after this many body bytes (never ends). */
stallAfter?: number
/** Serve different bytes of the same length. */
body?: Buffer
}
export interface Fixture {
origin: string
requests: RecordedRequest[]
gets(): RecordedRequest[]
behavior: FixtureBehavior
close(): Promise<void>
}
/**
* Keeps loopback fetches off any proxy configured in the ambient environment
* (a developer TUN/HTTP proxy would otherwise mangle fixture traffic).
* Returns a restore function.
*/
export function bypassProxyForLoopback(): () => void {
const saved = { NO_PROXY: process.env.NO_PROXY, no_proxy: process.env.no_proxy }
process.env.NO_PROXY = '127.0.0.1,localhost,::1'
process.env.no_proxy = '127.0.0.1,localhost,::1'
return () => {
for (const [key, value] of Object.entries(saved)) {
if (value === undefined) delete process.env[key]
else process.env[key] = value
}
}
}
export function makeContent(size: number, seed = 7): Buffer {
const buffer = Buffer.alloc(size)
let state = seed
for (let i = 0; i < size; i++) {
state = (state * 1103515245 + 12345) & 0x7fffffff
buffer[i] = state >> 16
}
return buffer
}
export function sha256Hex(data: Uint8Array): string {
return createHash('sha256').update(data).digest('hex')
}
export function assetFor(content: Buffer, urls: string[], name = 'fixture.bin'): DownloadAsset {
return { name, bytes: content.length, hash: { algorithm: 'sha256', encoding: 'hex', value: sha256Hex(content) }, urls }
}
export async function startFixture(content: Buffer, behavior: FixtureBehavior = {}): Promise<Fixture> {
const requests: RecordedRequest[] = []
let getCount = 0
const sockets = new Set<import('node:net').Socket>()
const handle = (request: IncomingMessage, response: ServerResponse): void => {
const method = request.method ?? 'GET'
const range = request.headers.range
requests.push({ method, range, path: request.url ?? '/' })
const payload = fixture.behavior.body ?? content
if (method === 'HEAD') {
const delay = fixture.behavior.headDelayMs ?? 0
setTimeout(() => {
response.writeHead(fixture.behavior.headStatus ?? 200, { 'content-length': String(payload.length) }).end()
}, delay)
return
}
const attempt = ++getCount
if (fixture.behavior.status && fixture.behavior.status !== 200) {
response.writeHead(fixture.behavior.status).end('nope')
return
}
if (range && fixture.behavior.rangeNotSatisfiable) {
response.writeHead(416, { 'content-range': `bytes */${payload.length}` }).end()
return
}
let start = 0
let status = 200
const match = /^bytes=(\d+)-$/.exec(range ?? '')
if (match && !fixture.behavior.ignoreRange) {
start = Number(match[1])
status = 206
}
const body = payload.subarray(start)
const headers: Record<string, string> = { 'content-length': String(body.length) }
if (status === 206) headers['content-range'] = `bytes ${start}-${payload.length - 1}/${payload.length}`
response.writeHead(status, headers)
const breakAfter = fixture.behavior.breakAfter
const breaking = breakAfter !== undefined && attempt <= (fixture.behavior.breakRequests ?? Infinity)
const stallAfter = fixture.behavior.stallAfter
const limit = breaking ? breakAfter : stallAfter !== undefined ? stallAfter : body.length
const chunkSize = 16 * 1024
let sent = 0
const pump = (): void => {
if (response.destroyed) return
if (sent >= limit) {
if (breaking) {
setTimeout(() => response.destroy(), 20)
} else if (stallAfter === undefined) {
response.end()
}
return
}
const chunk = body.subarray(sent, Math.min(sent + chunkSize, limit))
sent += chunk.length
response.write(chunk, () => {
const delay = fixture.behavior.chunkDelayMs ?? 0
if (delay > 0) setTimeout(pump, delay)
else setImmediate(pump)
})
}
pump()
}
const server = createServer(handle)
server.on('connection', socket => {
sockets.add(socket)
socket.once('close', () => sockets.delete(socket))
})
await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve))
const port = (server.address() as { port: number }).port
const fixture: Fixture = {
origin: `http://127.0.0.1:${port}`,
requests,
gets: () => requests.filter(request => request.method === 'GET'),
behavior,
close: () => new Promise<void>(resolve => {
for (const socket of sockets) socket.destroy()
server.close(() => resolve())
}),
}
return fixture
}
@@ -0,0 +1,14 @@
export {
DEFAULT_MAX_RETRIES,
defaultSleep,
downloadAsset,
partPathFor,
type DownloadAsset,
type DownloadHash,
type DownloadOptions,
type DownloadProgress,
type DownloadResult,
type FetchLike,
type HashAlgorithm,
} from './downloader.js'
export { VoiceDownloadError, classifyError, sanitizeOrigin } from './failure.js'
+24
View File
@@ -0,0 +1,24 @@
import type { VoiceErrorBody, VoiceErrorCode } from './types.js'
const STATUS_BY_CODE: Record<VoiceErrorCode, number> = {
'voice/invalid-audio': 400,
'voice/unknown-provider': 404,
'voice/not-ready': 409,
'voice/failed': 500,
}
export class VoiceServiceError extends Error {
readonly code: VoiceErrorCode
readonly status: number
constructor(code: VoiceErrorCode, message: string) {
super(message)
this.name = 'VoiceServiceError'
this.code = code
this.status = STATUS_BY_CODE[code]
}
toBody(): VoiceErrorBody {
return { error: this.code, message: this.message }
}
}
+30
View File
@@ -0,0 +1,30 @@
import { DEFAULT_VOICE_PREFERENCES, type VoiceLanguage, type VoicePreferences } from './types.js'
export const VOICE_LANGUAGES: readonly VoiceLanguage[] = ['auto', 'zh', 'en', 'ja', 'ko', 'yue']
const MAX_PROVIDER_ID_LENGTH = 80
export function isVoiceLanguage(value: unknown): value is VoiceLanguage {
return typeof value === 'string' && (VOICE_LANGUAGES as readonly string[]).includes(value)
}
/**
* Lenient read-side normalization for the `voiceInput` section of
* desktop-ui.json. Unknown fields are kept so a newer build's data survives a
* round trip through an older one.
*/
export function normalizeVoicePreferences(value: unknown): VoicePreferences {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
return { ...DEFAULT_VOICE_PREFERENCES }
}
const record = value as Record<string, unknown>
const providerId = typeof record.providerId === 'string' ? record.providerId.trim() : ''
return {
...record,
enabled: typeof record.enabled === 'boolean' ? record.enabled : DEFAULT_VOICE_PREFERENCES.enabled,
providerId: providerId.length > 0 && providerId.length <= MAX_PROVIDER_ID_LENGTH
? providerId
: DEFAULT_VOICE_PREFERENCES.providerId,
language: isVoiceLanguage(record.language) ? record.language : DEFAULT_VOICE_PREFERENCES.language,
}
}
@@ -0,0 +1,22 @@
import { DesktopUiPreferencesService } from '../desktopUiPreferencesService.js'
import type { VoicePreferences } from './types.js'
/** Where voiceService keeps the user's voice-input choices. Injected for tests. */
export interface VoicePreferencesStore {
read(): Promise<VoicePreferences>
update(patch: Partial<VoicePreferences>): Promise<VoicePreferences>
}
/** Persists preferences in the `voiceInput` section of `cc-haha/desktop-ui.json`. */
export function createDesktopUiVoicePreferencesStore(
service: DesktopUiPreferencesService = new DesktopUiPreferencesService(),
): VoicePreferencesStore {
return {
async read() {
return (await service.readPreferences()).preferences.voiceInput
},
async update(patch) {
return (await service.updateVoiceInputPreferences(patch)).voiceInput
},
}
}
+27
View File
@@ -0,0 +1,27 @@
import type { SpeechProvider } from './types.js'
/**
* Provider registry: a plain Map keyed by `info.id`. There is deliberately no
* fallback chain — a request names the provider it wants, or the saved
* preference does.
*/
export class VoiceProviderRegistry {
private readonly providers = new Map<string, SpeechProvider>()
register(provider: SpeechProvider): this {
const id = provider.info.id
if (this.providers.has(id)) {
throw new Error(`Voice provider already registered: ${id}`)
}
this.providers.set(id, provider)
return this
}
get(id: string): SpeechProvider | undefined {
return this.providers.get(id)
}
list(): SpeechProvider[] {
return [...this.providers.values()]
}
}
@@ -0,0 +1,87 @@
/** Tiny stand-ins for the runtime tarballs and model files, served from loopback fixtures. */
import { createHash } from 'node:crypto'
import { execFile } from 'node:child_process'
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { promisify } from 'node:util'
import type { InstallItem } from '../assets.js'
import { makeContent, startFixture, type Fixture, type FixtureBehavior } from '../../download/httpFixture.testUtil.js'
const run = promisify(execFile)
/** Builds an npm-style tarball (everything under `package/`). */
export async function makeTarball(files: Record<string, string>): Promise<Buffer> {
const work = await mkdtemp(join(tmpdir(), 'voice-tarball-'))
try {
for (const [name, content] of Object.entries(files)) {
const path = join(work, 'package', name)
await mkdir(dirname(path), { recursive: true })
await writeFile(path, content)
}
await run('tar', ['-czf', 'out.tgz', 'package'], { cwd: work })
return await readFile(join(work, 'out.tgz'))
} finally {
await rm(work, { recursive: true, force: true })
}
}
export interface FakeAsset {
item: InstallItem
content: Buffer
fixture: Fixture
}
export interface FakeAssets {
items: InstallItem[]
byTarget: Record<string, FakeAsset>
totalBytes: number
close(): Promise<void>
}
export async function startFakeAssets(
behaviors: Record<string, FixtureBehavior> = {},
overrides: Record<string, Buffer> = {},
): Promise<FakeAssets> {
const byTarget: Record<string, FakeAsset> = {}
const items: InstallItem[] = []
const add = async (item: Omit<InstallItem, 'asset'>, content: Buffer, algorithm: 'sha256' | 'sha512'): Promise<void> => {
const served = overrides[item.target] ?? content
const fixture = await startFixture(served, behaviors[item.target] ?? {})
const encoding = algorithm === 'sha256' ? 'hex' : 'base64'
const name = item.kind === 'package' ? `${item.target}-1.13.8.tgz` : item.target
const full: InstallItem = {
...item,
asset: {
name,
bytes: content.length,
hash: { algorithm, encoding, value: createHash(algorithm).update(content).digest(encoding) },
urls: [`${fixture.origin}/${name}`],
},
}
items.push(full)
byTarget[item.target] = { item: full, content, fixture }
}
await add(
{ step: 'runtime', kind: 'package', target: 'sherpa-onnx-node', marker: 'sherpa-onnx.js' },
await makeTarball({ 'sherpa-onnx.js': 'module.exports = {}', 'package.json': '{"name":"sherpa-onnx-node"}' }),
'sha512',
)
await add(
{ step: 'runtime', kind: 'package', target: 'sherpa-onnx-test-arch', marker: 'sherpa-onnx.node' },
await makeTarball({ 'sherpa-onnx.node': 'native-bytes', 'libonnxruntime.dylib': 'lib' }),
'sha512',
)
await add({ step: 'model', kind: 'file', target: 'model.int8.onnx' }, makeContent(120_000, 1), 'sha256')
await add({ step: 'model', kind: 'file', target: 'tokens.txt' }, makeContent(2_000, 2), 'sha256')
await add({ step: 'vad', kind: 'file', target: 'silero_vad.onnx' }, makeContent(30_000, 3), 'sha256')
return {
items,
byTarget,
totalBytes: items.reduce((sum, item) => sum + item.asset.bytes, 0),
close: async () => { await Promise.all(Object.values(byTarget).map(asset => asset.fixture.close())) },
}
}
@@ -0,0 +1,32 @@
import { mkdir, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
/**
* Writes a sherpa-onnx-node stand-in at the path the worker loads it from
* (`<root>/node_modules/sherpa-onnx-node/sherpa-onnx.js`). Transcripts read
* `<language>:<sample count>`.
*/
export async function installFakeSherpa(root: string): Promise<void> {
const packageDir = join(root, 'node_modules', 'sherpa-onnx-node')
await mkdir(packageDir, { recursive: true })
await writeFile(join(packageDir, 'sherpa-onnx.js'), `
let language = 'auto'
class OfflineRecognizer {
constructor(config) { this.config = config }
createStream() { return { samples: 0, acceptWaveform(a) { this.samples += a.samples.length } } }
setConfig(config) { language = config.modelConfig.senseVoice.language }
decode() {}
getResult(stream) { return { text: language + ':' + stream.samples } }
}
class Vad {
constructor() { this.parts = []; this.ready = [] }
acceptWaveform(samples) { this.parts.push(samples.length) }
isEmpty() { return this.ready.length === 0 }
front() { return { samples: new Float32Array(this.ready[0]) } }
pop() { this.ready.shift() }
reset() { this.parts = []; this.ready = [] }
flush() { this.ready.push(this.parts.reduce((a, b) => a + b, 0)) }
}
module.exports = { OfflineRecognizer, Vad }
`)
}
@@ -0,0 +1,46 @@
// Stand-in for the SenseVoice worker: real process and HTTP lifetimes, no sherpa.
// The requested language selects a behavior; marker files next to the model path
// (config.model's directory) steer startup and record what happened.
import { createServer } from 'node:http'
import { appendFileSync, existsSync } from 'node:fs'
import { dirname, join } from 'node:path'
const config = JSON.parse(process.env.CC_HAHA_VOICE_WORKER_CONFIG)
const token = process.env.CC_HAHA_VOICE_WORKER_TOKEN
delete process.env.CC_HAHA_VOICE_WORKER_CONFIG
delete process.env.CC_HAHA_VOICE_WORKER_TOKEN
const dir = dirname(config.model)
const log = (name, line) => appendFileSync(join(dir, name), `${line}\n`)
log('starts.log', String(process.pid))
if (existsSync(join(dir, 'fail-start'))) {
process.stderr.write('fake worker: model failed to load\n')
process.exit(3)
}
const startDelay = existsSync(join(dir, 'slow-start')) ? 200 : 0
const server = createServer((request, response) => {
if (request.headers.authorization !== `Bearer ${token}`) {
response.writeHead(401).end('{}')
return
}
const language = new URL(request.url, 'http://localhost').searchParams.get('language')
log('requests.log', `${process.pid} ${language}`)
request.resume()
request.on('end', () => {
if (language === 'hold') return
if (language === 'crash') process.exit(1)
const json = (status, body) => response.writeHead(status, { 'content-type': 'application/json' }).end(JSON.stringify(body))
if (language === 'invalid-input') return json(400, { error: 'invalid input', code: 'invalid-input' })
if (language === 'error') return json(500, { error: 'provider failed' })
if (language === 'garbage') return response.writeHead(200).end('not json')
const respond = () => json(200, { text: `echo:${language}`, audioSeconds: 1, inferenceSeconds: 0.01 })
if (language === 'slow') setTimeout(respond, 150)
else respond()
})
})
setTimeout(() => {
server.listen(0, '127.0.0.1', () => process.stdout.write(`${JSON.stringify({ port: server.address().port })}\n`))
}, startDelay)
process.stdin.resume()
process.stdin.on('end', () => process.exit(0))
@@ -0,0 +1,39 @@
/** Builds WAV bytes for tests. */
export function makeWav(
seconds: number,
options: { sampleRate?: number; channels?: number; bits?: number; encoding?: number; extraChunk?: boolean; dataSizeOverride?: number } = {},
): Uint8Array {
const sampleRate = options.sampleRate ?? 16_000
const channels = options.channels ?? 1
const bits = options.bits ?? 16
const bytesPerFrame = channels * (bits / 8)
const dataBytes = Math.round(seconds * sampleRate) * bytesPerFrame
const extra = options.extraChunk ? 8 + 4 : 0
const buffer = new Uint8Array(12 + 24 + extra + 8 + dataBytes)
const view = new DataView(buffer.buffer)
const ascii = (offset: number, text: string): void => {
for (let i = 0; i < text.length; i++) buffer[offset + i] = text.charCodeAt(i)
}
ascii(0, 'RIFF')
view.setUint32(4, buffer.length - 8, true)
ascii(8, 'WAVE')
ascii(12, 'fmt ')
view.setUint32(16, 16, true)
view.setUint16(20, options.encoding ?? 1, true)
view.setUint16(22, channels, true)
view.setUint32(24, sampleRate, true)
view.setUint32(28, sampleRate * bytesPerFrame, true)
view.setUint16(32, bytesPerFrame, true)
view.setUint16(34, bits, true)
let offset = 36
if (options.extraChunk) {
ascii(offset, 'LIST')
view.setUint32(offset + 4, 4, true)
ascii(offset + 8, 'INFO')
offset += 12
}
ascii(offset, 'data')
view.setUint32(offset + 4, options.dataSizeOverride ?? dataBytes, true)
for (let i = 0; i < dataBytes / 2; i++) view.setInt16(offset + 8 + i * 2, (i * 37) % 2000 - 1000, true)
return buffer
}
@@ -0,0 +1,135 @@
/**
* Pinned SenseVoice assets: the sherpa-onnx runtime (npm tarballs) and the
* INT8 model files. Sizes and digests were taken from the registry and the
* model repository; both registries and both model hosts serve identical bytes.
*/
import type { DownloadAsset } from '../download/index.js'
export const SHERPA_VERSION = '1.13.8'
export type RuntimePlatform = 'darwin-arm64' | 'darwin-x64' | 'linux-x64' | 'linux-arm64' | 'win-x64'
const NPM_REGISTRIES = ['https://registry.npmjs.org', 'https://registry.npmmirror.com']
const MODEL_REVISION = '2365baeacb507f821a0c8120fcee3d484dba7a07'
const VAD_REVISION = 'fba88cd2e921609e7675c3aaf51e0b9b295da4bc'
const MODEL_ORIGINS = ['https://huggingface.co', 'https://hf-mirror.com']
/** Runtime packages keyed by the platform they load on. Values are npm sha512 integrity. */
const RUNTIME_PACKAGES: Record<'sherpa-onnx-node' | RuntimePlatform, { bytes: number; integrity: string }> = {
'sherpa-onnx-node': {
bytes: 11_954,
integrity: 'MsDMBdhLFTZ1GwvcGSSQhnS7g/EA8OMH6IYysCVUOM7j8Icty9KRc0E6YT1A5fWBsZwRfKOeh88QC95aRvS8ag==',
},
'darwin-arm64': {
bytes: 10_047_754,
integrity: 'FPNgJMgnWVl/KhRTIhG3KL3A4Om63Rn4YKXc9/uHY7SzLcvqLJLc/h7UBWJwduXvv7K18t5NpxHR6XgXn4sjWw==',
},
'darwin-x64': {
bytes: 11_191_481,
integrity: '7BLRpjM6w4f9W46/nmkmq8lEKUayhebvcpslCVQ+6QN2uReYlZEMDZlSpXMjme+hUFrPfRz8P3UNq8ep/4d19g==',
},
'linux-x64': {
bytes: 11_089_653,
integrity: '6plnhjagsSeTntCgnlag86hWbs/uZE9Crms1LgOb68/1nKsIQjMd+WG519m+aPwT6TrsBOiEMzrx41t8sL5L5g==',
},
'linux-arm64': {
bytes: 13_910_679,
integrity: 'Tlg7a70b/Wge3OF8IgTHF9jhSVCsLyKQKhwc4BsJ5A+dL/SrFtGBjzuHp4XeLhiiOT7afCxX5PdSn/D4c8Lnuw==',
},
'win-x64': {
bytes: 8_894_875,
integrity: 'oZF1c9VPOKtMwn83Bboc5XSWL+76BRoyB3eUuVnCknBKxwSULZU2Foia9VHWzU+n4I12rPsP6z6H9Rp1hD9o8g==',
},
}
export type InstallStep = 'runtime' | 'model' | 'vad'
/** One thing prepare() installs: a plain file or an npm package extracted into node_modules. */
export interface InstallItem {
step: InstallStep
kind: 'file' | 'package'
asset: DownloadAsset
/** Package directory name for `package` items, file name inside `models/` for `file` items. */
target: string
/** For packages: a file that must exist after extraction and proves the layout. */
marker?: string
}
export function resolveRuntimePlatform(
platform: NodeJS.Platform = process.platform,
arch: string = process.arch,
): RuntimePlatform | undefined {
const key = `${platform === 'win32' ? 'win' : platform}-${arch}`
return key in RUNTIME_PACKAGES && key !== 'sherpa-onnx-node' ? (key as RuntimePlatform) : undefined
}
function npmAsset(pkg: string, bytes: number, integrity: string): DownloadAsset {
const file = `${pkg}-${SHERPA_VERSION}.tgz`
return {
name: file,
bytes,
hash: { algorithm: 'sha512', encoding: 'base64', value: integrity },
urls: NPM_REGISTRIES.map(registry => `${registry}/${pkg}/-/${file}`),
}
}
function modelAsset(
name: string,
bytes: number,
sha256: string,
repository: string,
revision: string,
): DownloadAsset {
return {
name,
bytes,
hash: { algorithm: 'sha256', encoding: 'hex', value: sha256 },
urls: MODEL_ORIGINS.map(origin => `${origin}/${repository}/resolve/${revision}/${name}`),
}
}
export function runtimeItems(platform: RuntimePlatform): InstallItem[] {
const node = RUNTIME_PACKAGES['sherpa-onnx-node']
const native = RUNTIME_PACKAGES[platform]
return [
{
step: 'runtime', kind: 'package', target: 'sherpa-onnx-node', marker: 'sherpa-onnx.js',
asset: npmAsset('sherpa-onnx-node', node.bytes, node.integrity),
},
{
step: 'runtime', kind: 'package', target: `sherpa-onnx-${platform}`, marker: 'sherpa-onnx.node',
asset: npmAsset(`sherpa-onnx-${platform}`, native.bytes, native.integrity),
},
]
}
export function modelItems(): InstallItem[] {
const repository = 'csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17'
return [
{
step: 'model', kind: 'file', target: 'model.int8.onnx',
asset: modelAsset('model.int8.onnx', 239_233_841,
'c71f0ce00bec95b07744e116345e33d8cbbe08cef896382cf907bf4b51a2cd51', repository, MODEL_REVISION),
},
{
step: 'model', kind: 'file', target: 'tokens.txt',
asset: modelAsset('tokens.txt', 315_894,
'f449eb28dc567533d7fa59be34e2abca8784f771850c78a47fb731a31429a1dc', repository, MODEL_REVISION),
},
{
step: 'vad', kind: 'file', target: 'silero_vad.onnx',
asset: modelAsset('silero_vad.onnx', 1_807_522,
'a35ebf52fd3ce5f1469b2a36158dba761bc47b973ea3382b3186ca15b1f5af28', 'csukuangfj/vad', VAD_REVISION),
},
]
}
/** Everything a platform needs, in install order. Unsupported platforms get models only. */
export function installItems(platform: RuntimePlatform | undefined): InstallItem[] {
return [...(platform ? runtimeItems(platform) : []), ...modelItems()]
}
export function totalDownloadBytes(items: InstallItem[]): number {
return items.reduce((sum, item) => sum + item.asset.bytes, 0)
}
@@ -0,0 +1,8 @@
export {
SENSEVOICE_PROVIDER_ID,
createSenseVoiceProvider,
type SenseVoiceProvider,
type SenseVoiceProviderOptions,
} from './provider.js'
export { resolveRuntimePlatform, type RuntimePlatform } from './assets.js'
export { runVoiceWorker } from './worker.js'
@@ -0,0 +1,68 @@
import { afterEach, beforeEach, describe, expect, it } from 'bun:test'
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { VoiceDownloadError } from '../download/index.js'
import { makeTarball } from './__fixtures__/fakeAssets.js'
import { extractWithSystemTar, removeInstall } from './install.js'
import { senseVoiceLayout, type SenseVoiceLayout } from './layout.js'
let dir: string
let layout: SenseVoiceLayout
beforeEach(async () => {
dir = await mkdtemp(join(tmpdir(), 'voice-install-'))
layout = senseVoiceLayout(dir)
})
afterEach(async () => {
await rm(dir, { recursive: true, force: true })
})
describe('extractWithSystemTar', () => {
it('unpacks an npm tarball without its package/ prefix, using paths relative to cwd', async () => {
await mkdir(join(dir, 'downloads'), { recursive: true })
await mkdir(join(dir, 'staging', 'pkg'), { recursive: true })
await writeFile(join(dir, 'downloads', 'pkg.tgz'), await makeTarball({ 'sherpa-onnx.js': 'entry', 'lib/deep.txt': 'deep' }))
await extractWithSystemTar({
cwd: dir,
archive: 'downloads/pkg.tgz',
into: 'staging/pkg',
signal: new AbortController().signal,
})
expect(await readFile(join(dir, 'staging', 'pkg', 'sherpa-onnx.js'), 'utf8')).toBe('entry')
expect(await readFile(join(dir, 'staging', 'pkg', 'lib', 'deep.txt'), 'utf8')).toBe('deep')
expect(await stat(join(dir, 'staging', 'pkg', 'package')).catch(() => undefined)).toBeUndefined()
})
it('reports a corrupt archive as a storage failure', async () => {
await mkdir(join(dir, 'staging'), { recursive: true })
await writeFile(join(dir, 'broken.tgz'), 'not a tarball')
const error = await extractWithSystemTar({
cwd: dir, archive: 'broken.tgz', into: 'staging', signal: new AbortController().signal,
}).catch(e => e)
expect(error).toBeInstanceOf(VoiceDownloadError)
expect(error.failure.reason).toBe('storage')
})
it('stops when cancelled before it starts', async () => {
const controller = new AbortController()
controller.abort(new Error('cancelled'))
await expect(extractWithSystemTar({ cwd: dir, archive: 'x.tgz', into: '.', signal: controller.signal }))
.rejects.toThrow('cancelled')
})
})
describe('removeInstall', () => {
it('removes everything under the provider directory and tolerates a missing one', async () => {
await mkdir(layout.models, { recursive: true })
await writeFile(join(layout.models, 'a'), 'a')
await removeInstall(layout)
expect(await stat(layout.base).catch(() => undefined)).toBeUndefined()
await removeInstall(layout)
})
})
@@ -0,0 +1,248 @@
/**
* Installs the SenseVoice runtime and model into the data directory.
*
* Every item is downloaded with resume/retry/mirror fallback, verified against
* its pinned digest, and only then recorded in `verified.json`. Readiness is a
* cheap check against that record, so nothing is re-hashed on startup.
*/
import { spawn } from 'node:child_process'
import { mkdir, rename, rm, stat } from 'node:fs/promises'
import { dirname, join, relative } from 'node:path'
import {
VoiceDownloadError,
classifyError,
downloadAsset,
partPathFor,
type DownloadOptions,
} from '../download/index.js'
import type { VoicePreparationState } from '../types.js'
import { totalDownloadBytes, type InstallItem } from './assets.js'
import {
itemDownloadPath,
itemManifestName,
itemPath,
itemProofPath,
type SenseVoiceLayout,
} from './layout.js'
import {
describeFile,
forgetVerified,
hashFile,
matchesRecord,
readManifest,
recordVerified,
} from './manifest.js'
export type ExtractArchive = (input: {
/** Working directory; both paths below are relative to it and use forward slashes. */
cwd: string
archive: string
into: string
signal: AbortSignal
}) => Promise<void>
export interface InstallContext {
layout: SenseVoiceLayout
items: InstallItem[]
download: DownloadOptions
extract?: ExtractArchive
}
function toPosix(path: string): string {
return path.split(/[\\/]/).join('/')
}
/** Extracts an npm tarball, dropping its leading `package/` directory, using the system `tar`. */
export const extractWithSystemTar: ExtractArchive = ({ cwd, archive, into, signal }) =>
new Promise((resolve, reject) => {
// Relative paths keep GNU tar on Windows from reading `C:\...` as a remote host.
const child = spawn('tar', ['-xzf', archive, '-C', into, '--strip-components=1'], {
cwd,
stdio: ['ignore', 'ignore', 'pipe'],
windowsHide: true,
})
let stderr = ''
child.stderr.setEncoding('utf8')
child.stderr.on('data', (chunk: string) => { stderr = (stderr + chunk).slice(-2000) })
const abort = (): void => {
child.kill('SIGKILL')
reject(signal.reason)
}
if (signal.aborted) {
abort()
return
}
signal.addEventListener('abort', abort, { once: true })
child.once('error', error => {
signal.removeEventListener('abort', abort)
const missing = (error as NodeJS.ErrnoException).code === 'ENOENT'
reject(new VoiceDownloadError({
reason: 'storage',
resource: archive,
message: missing ? 'The system "tar" command is required to unpack the speech runtime' : error.message,
}, { cause: error }))
})
child.once('exit', code => {
signal.removeEventListener('abort', abort)
if (code === 0) resolve()
else {
reject(new VoiceDownloadError({
reason: 'storage',
resource: archive,
message: `Could not unpack ${archive} (tar exit ${code}): ${stderr.trim()}`,
}))
}
})
})
async function installPackage(
ctx: InstallContext,
item: InstallItem,
archivePath: string,
signal: AbortSignal,
): Promise<void> {
const { layout } = ctx
const staging = join(layout.staging, item.target)
await rm(staging, { recursive: true, force: true })
await mkdir(staging, { recursive: true })
try {
await (ctx.extract ?? extractWithSystemTar)({
cwd: layout.base,
archive: toPosix(relative(layout.base, archivePath)),
into: toPosix(relative(layout.base, staging)),
signal,
})
const marker = await stat(join(staging, item.marker!)).catch(() => undefined)
if (!marker?.isFile()) {
throw new VoiceDownloadError({
reason: 'integrity',
resource: item.asset.name,
message: `${item.asset.name} does not contain ${item.marker}`,
})
}
const destination = itemPath(layout, item)
await rm(destination, { recursive: true, force: true })
await mkdir(dirname(destination), { recursive: true })
await rename(staging, destination)
} finally {
await rm(staging, { recursive: true, force: true }).catch(() => {})
}
await rm(archivePath, { force: true }).catch(() => {})
}
/** True when every item is recorded as verified and its file still matches the record. */
export async function isInstalled(layout: SenseVoiceLayout, items: InstallItem[]): Promise<boolean> {
const recorded = new Map((await readManifest(layout.manifest)).map(file => [file.name, file]))
for (const item of items) {
const entry = recorded.get(itemManifestName(layout, item))
const expectedBytes = item.kind === 'file' ? item.asset.bytes : undefined
if (!await matchesRecord(entry, itemProofPath(layout, item), expectedBytes)) return false
}
return true
}
/** Bytes present on disk (installed items plus partial downloads), for resume hints. */
export async function bytesOnDisk(layout: SenseVoiceLayout, items: InstallItem[]): Promise<number> {
const recorded = new Map((await readManifest(layout.manifest)).map(file => [file.name, file]))
let total = 0
for (const item of items) {
const entry = recorded.get(itemManifestName(layout, item))
if (await matchesRecord(entry, itemProofPath(layout, item), item.kind === 'file' ? item.asset.bytes : undefined)) {
total += item.asset.bytes
continue
}
total += await stat(partPathFor(itemDownloadPath(layout, item))).then(info => info.size, () => 0)
}
return total
}
type Report = (state: VoicePreparationState) => void
/** Installs whatever is missing. Throws VoiceDownloadError on failure; abort errors pass through. */
export async function installAll(ctx: InstallContext, signal: AbortSignal, report: Report): Promise<void> {
const { layout, items } = ctx
const totalBytes = totalDownloadBytes(items)
const recorded = new Map((await readManifest(layout.manifest)).map(file => [file.name, file]))
let base = 0
for (const item of items) {
signal.throwIfAborted()
const name = itemManifestName(layout, item)
const proof = itemProofPath(layout, item)
const expectedBytes = item.kind === 'file' ? item.asset.bytes : undefined
const current = { step: item.step, resource: item.asset.name } as const
if (await matchesRecord(recorded.get(name), proof, expectedBytes)) {
base += item.asset.bytes
continue
}
// A complete file from a run that died before it could be recorded only needs its digest checked.
if (item.kind === 'file' && await stat(proof).then(info => info.size === item.asset.bytes, () => false)) {
report({ phase: 'verifying', ...current, completedBytes: base, totalBytes })
if (await hashFile(proof) === item.asset.hash.value) {
await recordVerified(layout.manifest, [await describeFile(name, proof, item.asset.hash.value)])
base += item.asset.bytes
continue
}
await rm(proof, { force: true })
}
report({ phase: 'downloading', ...current, completedBytes: base, totalBytes })
await downloadAsset(item.asset, itemDownloadPath(layout, item), {
...ctx.download,
signal,
onProgress: progress => {
report({
phase: 'downloading',
...current,
completedBytes: base + progress.completedBytes,
totalBytes,
source: progress.source,
...(progress.resumedFromBytes !== undefined ? { resumedFromBytes: base + progress.resumedFromBytes } : {}),
})
},
})
if (item.kind === 'package') {
report({ phase: 'verifying', ...current, completedBytes: base + item.asset.bytes, totalBytes })
await installPackage(ctx, item, itemDownloadPath(layout, item), signal)
await recordVerified(layout.manifest, [await describeFile(name, proof)])
} else {
await recordVerified(layout.manifest, [await describeFile(name, proof, item.asset.hash.value)])
}
base += item.asset.bytes
}
report({ phase: 'verifying', step: 'verify', completedBytes: totalBytes, totalBytes })
if (!await isInstalled(layout, items)) {
throw new VoiceDownloadError({
reason: 'integrity',
message: 'Installed speech files did not pass the final check',
})
}
}
export async function removeInstall(layout: SenseVoiceLayout): Promise<void> {
await forgetVerified(layout.manifest)
await rm(layout.base, { recursive: true, force: true })
}
/** Converts any thrown value into the failure state shown to the user. */
export function failureState(
error: unknown,
fallback: Pick<VoicePreparationState, 'step' | 'resource' | 'completedBytes' | 'totalBytes'>,
): VoicePreparationState {
const failure = error instanceof VoiceDownloadError
? error.failure
: {
reason: classifyError(error),
message: error instanceof Error ? error.message : String(error),
}
return {
phase: 'failed',
...fallback,
...(failure.source ? { source: failure.source } : {}),
error: failure,
}
}
@@ -0,0 +1,55 @@
import { join } from 'node:path'
import { getCcHahaDir } from '../../../../utils/envUtils.js'
import { SHERPA_VERSION, type InstallItem } from './assets.js'
/** `<CLAUDE_CONFIG_DIR>/cc-haha/voice`, resolved lazily so tests can redirect the config dir. */
export function defaultVoiceDataRoot(): string {
return join(getCcHahaDir(), 'voice')
}
export interface SenseVoiceLayout {
/** `<dataRoot>/sensevoice`; deleting it removes everything this provider installed. */
base: string
models: string
/** Deterministic `.part` files live here, so an interrupted download can continue. */
downloads: string
staging: string
/** Directory whose `node_modules` holds sherpa-onnx-node and its native package. */
runtimeRoot: string
manifest: string
}
export function senseVoiceLayout(dataRoot: string): SenseVoiceLayout {
const base = join(dataRoot, 'sensevoice')
return {
base,
models: join(base, 'models'),
downloads: join(base, 'downloads'),
staging: join(base, 'staging'),
runtimeRoot: join(base, 'runtime', SHERPA_VERSION),
manifest: join(base, 'verified.json'),
}
}
/** Where an item's final payload lives (file path, or package directory). */
export function itemPath(layout: SenseVoiceLayout, item: InstallItem): string {
return item.kind === 'package'
? join(layout.runtimeRoot, 'node_modules', item.target)
: join(layout.models, item.target)
}
/** Destination of the downloaded bytes; for packages this is the tarball. */
export function itemDownloadPath(layout: SenseVoiceLayout, item: InstallItem): string {
return item.kind === 'package' ? join(layout.downloads, item.asset.name) : join(layout.models, item.target)
}
/** File whose size and mtime prove the item is still installed. */
export function itemProofPath(layout: SenseVoiceLayout, item: InstallItem): string {
return item.kind === 'package' ? join(itemPath(layout, item), item.marker!) : itemPath(layout, item)
}
/** Manifest key: forward-slash path relative to `base`. */
export function itemManifestName(layout: SenseVoiceLayout, item: InstallItem): string {
const relative = itemProofPath(layout, item).slice(layout.base.length + 1)
return relative.split(/[\\/]/).join('/')
}
@@ -0,0 +1,86 @@
/**
* `verified.json`: proof that installed files passed their digest check.
*
* Readiness never re-hashes the 240 MB model. A file counts as installed while
* it exists with the size recorded right after verification. mtime is ignored on
* purpose: backups, copies and sync tools rewrite it on intact files.
*/
import { createHash } from 'node:crypto'
import { createReadStream } from 'node:fs'
import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'
import { dirname } from 'node:path'
export interface VerifiedFile {
/** Forward-slash path relative to the provider base directory. */
name: string
bytes: number
sha256: string
}
interface Manifest {
version: 1
files: VerifiedFile[]
}
export async function readManifest(path: string): Promise<VerifiedFile[]> {
try {
const parsed = JSON.parse(await readFile(path, 'utf8')) as Partial<Manifest>
if (parsed?.version !== 1 || !Array.isArray(parsed.files)) return []
return parsed.files.filter((file): file is VerifiedFile =>
typeof file?.name === 'string'
&& Number.isFinite(file.bytes)
&& typeof file.sha256 === 'string')
} catch {
return []
}
}
/** Replaces entries by name; other entries survive. Written atomically. */
export async function recordVerified(path: string, entries: VerifiedFile[]): Promise<void> {
const names = new Set(entries.map(entry => entry.name))
const kept = (await readManifest(path)).filter(file => !names.has(file.name))
const manifest: Manifest = { version: 1, files: [...kept, ...entries] }
await mkdir(dirname(path), { recursive: true })
const temp = `${path}.tmp`
await writeFile(temp, `${JSON.stringify(manifest, null, 2)}\n`)
await rename(temp, path)
}
export async function forgetVerified(path: string): Promise<void> {
await rm(path, { force: true })
}
/** True when the recorded entry still describes the file on disk. */
export async function matchesRecord(
entry: VerifiedFile | undefined,
filePath: string,
expectedBytes?: number,
): Promise<boolean> {
if (!entry) return false
if (expectedBytes !== undefined && entry.bytes !== expectedBytes) return false
try {
const info = await stat(filePath)
return info.isFile() && info.size === entry.bytes
} catch {
return false
}
}
export async function describeFile(name: string, filePath: string, sha256?: string): Promise<VerifiedFile> {
const info = await stat(filePath)
return {
name,
bytes: info.size,
sha256: sha256 ?? await hashFile(filePath),
}
}
export function hashFile(filePath: string): Promise<string> {
return new Promise((resolve, reject) => {
const hash = createHash('sha256')
const stream = createReadStream(filePath)
stream.on('data', chunk => hash.update(chunk))
stream.once('error', reject)
stream.once('end', () => resolve(hash.digest('hex')))
})
}
@@ -0,0 +1,41 @@
/**
* Contract between the server-side recognizer and its worker process.
*
* The worker listens on 127.0.0.1 with a random port, prints `{"port":N}` as
* its first stdout line once the model is loaded, and answers
* `POST /transcribe?language=<code>` (WAV body, `Authorization: Bearer <token>`).
* The token and configuration arrive through environment variables that the
* worker deletes as soon as it has read them.
*/
export const WORKER_CONFIG_ENV = 'CC_HAHA_VOICE_WORKER_CONFIG'
export const WORKER_TOKEN_ENV = 'CC_HAHA_VOICE_WORKER_TOKEN'
export const WORKER_LANGUAGES: readonly string[] = ['auto', 'zh', 'en', 'ja', 'ko', 'yue']
export interface WorkerConfig {
/** Directory whose `node_modules` contains sherpa-onnx-node and its native package. */
runtimeRoot: string
model: string
tokens: string
vad: string
threads: number
maxAudioBytes: number
vadThreshold: number
minSilenceSeconds: number
minSpeechSeconds: number
/** Longest single VAD segment handed to the recognizer. */
segmentSeconds: number
}
export interface WorkerTranscript {
text: string
audioSeconds: number
inferenceSeconds: number
}
/** JSON error body; `code: 'invalid-input'` marks a request the worker rejected before inference. */
export interface WorkerErrorBody {
error: string
code?: 'invalid-input'
}
@@ -0,0 +1,387 @@
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'bun:test'
import { spawn } from 'node:child_process'
import { mkdtemp, readFile, readdir, rm, stat, truncate, utimes, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { partPathFor } from '../download/index.js'
import { bypassProxyForLoopback, makeContent } from '../download/httpFixture.testUtil.js'
import { VoiceServiceError } from '../errors.js'
import type { VoicePreparationState } from '../types.js'
import { makeWav } from './__fixtures__/wav.js'
import { startFakeAssets, type FakeAssets } from './__fixtures__/fakeAssets.js'
import { installItems, totalDownloadBytes } from './assets.js'
import { createSenseVoiceProvider, type SenseVoiceProvider, type SenseVoiceProviderOptions } from './provider.js'
import { senseVoiceLayout } from './layout.js'
import type { SpawnWorker } from './recognizer.js'
const fakeWorker = fileURLToPath(new URL('./__fixtures__/fakeWorker.mjs', import.meta.url))
const spawnFake: SpawnWorker = ({ env, cwd }) => spawn(process.execPath, [fakeWorker], { env, cwd, stdio: 'pipe' })
const noSleep = async () => {}
let restoreProxyEnv: () => void
let dataRoot: string
let assets: FakeAssets | undefined
let providers: SenseVoiceProvider[]
beforeAll(() => { restoreProxyEnv = bypassProxyForLoopback() })
afterAll(() => { restoreProxyEnv() })
beforeEach(async () => {
dataRoot = await mkdtemp(join(tmpdir(), 'voice-provider-'))
providers = []
})
afterEach(async () => {
await Promise.all(providers.map(provider => provider.dispose()))
await assets?.close()
assets = undefined
await rm(dataRoot, { recursive: true, force: true })
})
async function setup(
behaviors: Parameters<typeof startFakeAssets>[0] = {},
overrides: Partial<SenseVoiceProviderOptions> = {},
tampered: Parameters<typeof startFakeAssets>[1] = {},
): Promise<SenseVoiceProvider> {
assets = await startFakeAssets(behaviors, tampered)
return make(overrides)
}
function make(overrides: Partial<SenseVoiceProviderOptions> = {}): SenseVoiceProvider {
const provider = createSenseVoiceProvider({
dataRoot,
items: assets!.items,
fetch: (input, init) => fetch(input, init as RequestInit),
sleep: noSleep,
spawnWorker: spawnFake,
download: { maxRetries: 0, progressIntervalMs: 0, probeTimeoutMs: 100 },
...overrides,
})
providers.push(provider)
return provider
}
async function prepareCollecting(provider: SenseVoiceProvider, signal = new AbortController().signal) {
const states: VoicePreparationState[] = []
const error = await provider.preparation.prepare(signal, state => states.push(state)).then(() => undefined, e => e)
return { states, error }
}
const exists = (path: string) => stat(path).then(() => true, () => false)
describe('provider info and pinned assets', () => {
it('describes the local SenseVoice provider with the size of what it downloads', () => {
const provider = createSenseVoiceProvider({ dataRoot, platform: { platform: 'darwin', arch: 'arm64' } })
providers.push(provider)
expect(provider.info).toEqual({
id: 'sensevoice-local',
name: 'SenseVoice Small (INT8)',
location: 'local',
languages: ['auto', 'zh', 'en', 'ja', 'ko', 'yue'],
// sherpa-onnx-node + sherpa-onnx-darwin-arm64 + model.int8.onnx + tokens.txt + silero_vad.onnx
downloadBytes: 11_954 + 10_047_754 + 239_233_841 + 315_894 + 1_807_522,
})
})
it.each([
['darwin', 'arm64', 'sherpa-onnx-darwin-arm64'],
['darwin', 'x64', 'sherpa-onnx-darwin-x64'],
['linux', 'x64', 'sherpa-onnx-linux-x64'],
['linux', 'arm64', 'sherpa-onnx-linux-arm64'],
['win32', 'x64', 'sherpa-onnx-win-x64'],
] as const)('maps %s/%s to %s and pins every download', (platform, arch, nativePackage) => {
const provider = createSenseVoiceProvider({ dataRoot, platform: { platform, arch } })
providers.push(provider)
const platformKey = nativePackage.replace('sherpa-onnx-', '')
const items = installItems(platformKey as Parameters<typeof installItems>[0])
expect(items.map(item => item.target)).toEqual([
'sherpa-onnx-node', nativePackage, 'model.int8.onnx', 'tokens.txt', 'silero_vad.onnx',
])
expect(provider.info.downloadBytes).toBe(totalDownloadBytes(items))
for (const { asset } of items) {
expect(asset.bytes).toBeGreaterThan(0)
expect(asset.urls.length).toBeGreaterThanOrEqual(2)
expect(new Set(asset.urls.map(url => new URL(url).origin)).size).toBe(asset.urls.length)
if (asset.name.endsWith('.tgz')) {
expect(asset.hash).toMatchObject({ algorithm: 'sha512', encoding: 'base64' })
expect(asset.hash.value).toMatch(/^[A-Za-z0-9+/]{86}==$/)
expect(asset.urls[0]).toBe(`https://registry.npmjs.org/${asset.name.replace('-1.13.8.tgz', '')}/-/${asset.name}`)
expect(asset.urls[1]).toStartWith('https://registry.npmmirror.com/')
} else {
expect(asset.hash).toMatchObject({ algorithm: 'sha256', encoding: 'hex' })
expect(asset.hash.value).toMatch(/^[a-f0-9]{64}$/)
expect(asset.urls[0]).toStartWith('https://huggingface.co/')
expect(asset.urls[1]).toStartWith('https://hf-mirror.com/')
}
}
})
it('reports an unsupported platform instead of pretending it can download', async () => {
const provider = createSenseVoiceProvider({ dataRoot, platform: { platform: 'freebsd', arch: 'x64' } })
providers.push(provider)
expect(await provider.preparation.status()).toMatchObject({ phase: 'failed', error: { reason: 'unsupported-platform' } })
const states: VoicePreparationState[] = []
const error = await provider.preparation.prepare(new AbortController().signal, state => states.push(state)).catch(e => e)
expect(error).toBeInstanceOf(VoiceServiceError)
expect(states.at(-1)).toMatchObject({ phase: 'failed', error: { reason: 'unsupported-platform' } })
await expect(provider.transcribe(makeWav(1), { language: 'zh' }, new AbortController().signal))
.rejects.toMatchObject({ code: 'voice/not-ready' })
})
})
describe('prepare', () => {
it('installs runtime, model and VAD in order and ends ready', async () => {
const provider = await setup()
expect(await provider.preparation.status()).toEqual({ phase: 'unprepared' })
const { states, error } = await prepareCollecting(provider)
expect(error).toBeUndefined()
const sequence = states.map(state => `${state.phase}:${state.step ?? ''}`)
const firstOf = (marker: string) => sequence.indexOf(marker)
expect(firstOf('downloading:runtime')).toBeLessThan(firstOf('downloading:model'))
expect(firstOf('downloading:model')).toBeLessThan(firstOf('downloading:vad'))
expect(firstOf('downloading:vad')).toBeLessThan(firstOf('verifying:verify'))
expect(states.at(-1)).toEqual({ phase: 'ready' })
const downloading = states.filter(state => state.phase === 'downloading')
expect(downloading.every(state => state.totalBytes === assets!.totalBytes)).toBe(true)
const completed = downloading.map(state => state.completedBytes!)
expect([...completed].sort((a, b) => a - b)).toEqual(completed)
expect(downloading.at(-1)!.completedBytes).toBe(assets!.totalBytes)
expect(downloading.some(state => state.resource === 'model.int8.onnx' && state.source?.startsWith('http://127.0.0.1'))).toBe(true)
const layout = senseVoiceLayout(dataRoot)
expect(await readFile(join(layout.models, 'tokens.txt'))).toEqual(assets!.byTarget['tokens.txt']!.content)
expect(await readFile(join(layout.runtimeRoot, 'node_modules', 'sherpa-onnx-node', 'sherpa-onnx.js'), 'utf8')).toBe('module.exports = {}')
expect(await readFile(join(layout.runtimeRoot, 'node_modules', 'sherpa-onnx-test-arch', 'sherpa-onnx.node'), 'utf8')).toBe('native-bytes')
// No tarballs or staging directories are left behind.
expect(await readdir(layout.downloads)).toEqual([])
expect(await readdir(layout.staging)).toEqual([])
expect(await provider.preparation.status()).toEqual({ phase: 'ready' })
})
it('answers readiness from the verified record without touching the network or rehashing', async () => {
const provider = await setup()
await prepareCollecting(provider)
await assets!.close()
expect(await provider.preparation.status()).toEqual({ phase: 'ready' })
// A second prepare is a no-op that still ends ready.
const again = await prepareCollecting(provider)
expect(again.error).toBeUndefined()
expect(again.states.at(-1)).toEqual({ phase: 'ready' })
expect(again.states.some(state => state.phase === 'downloading')).toBe(false)
})
it('stays ready when only a file mtime changes (backup, copy or sync tools rewrite it)', async () => {
const provider = await setup()
await prepareCollecting(provider)
const layout = senseVoiceLayout(dataRoot)
const future = new Date(Date.now() + 60_000)
await utimes(join(layout.models, 'model.int8.onnx'), future, future)
await utimes(join(layout.runtimeRoot, 'node_modules', 'sherpa-onnx-node', 'sherpa-onnx.js'), future, future)
expect(await provider.preparation.status()).toEqual({ phase: 'ready' })
})
it('treats a truncated, deleted or unrecorded file as not installed and repairs it', async () => {
const provider = await setup()
await prepareCollecting(provider)
const layout = senseVoiceLayout(dataRoot)
const model = join(layout.models, 'model.int8.onnx')
await truncate(model, 100)
expect((await provider.preparation.status()).phase).toBe('unprepared')
await prepareCollecting(provider)
expect(await provider.preparation.status()).toEqual({ phase: 'ready' })
expect(await readFile(model)).toEqual(assets!.byTarget['model.int8.onnx']!.content)
await rm(join(layout.models, 'tokens.txt'))
expect((await provider.preparation.status()).phase).toBe('unprepared')
await prepareCollecting(provider)
expect(await provider.preparation.status()).toEqual({ phase: 'ready' })
await rm(layout.manifest)
expect((await provider.preparation.status()).phase).toBe('unprepared')
// The files are intact, so re-recording them must not download anything again.
const gets = assets!.byTarget['model.int8.onnx']!.fixture.gets().length
await prepareCollecting(provider)
expect(assets!.byTarget['model.int8.onnx']!.fixture.gets().length).toBe(gets)
})
it('keeps progress across an interrupted download and resumes from the partial file', async () => {
const provider = await setup({ 'model.int8.onnx': { breakAfter: 50_000, chunkDelayMs: 2 } })
const first = await prepareCollecting(provider)
expect(first.states.at(-1)).toMatchObject({
phase: 'failed', step: 'model', resource: 'model.int8.onnx', error: { reason: 'network' },
})
expect(first.error).toBeDefined()
const layout = senseVoiceLayout(dataRoot)
const partial = (await stat(partPathFor(join(layout.models, 'model.int8.onnx')))).size
expect(partial).toBeGreaterThan(0)
// The partial bytes are visible before the user clicks download again.
const status = await provider.preparation.status()
expect(status).toMatchObject({ phase: 'unprepared', totalBytes: assets!.totalBytes })
expect(status.completedBytes).toBeGreaterThan(partial)
assets!.byTarget['model.int8.onnx']!.fixture.behavior.breakAfter = undefined
const second = await prepareCollecting(provider)
expect(second.error).toBeUndefined()
const resumed = second.states.find(state => state.resumedFromBytes !== undefined)
const runtimeBytes = assets!.items.slice(0, 2).reduce((sum, item) => sum + item.asset.bytes, 0)
expect(resumed?.resumedFromBytes).toBe(runtimeBytes + partial)
expect(assets!.byTarget['model.int8.onnx']!.fixture.gets().at(-1)?.range).toBe(`bytes=${partial}-`)
expect(await readFile(join(layout.models, 'model.int8.onnx'))).toEqual(assets!.byTarget['model.int8.onnx']!.content)
expect(await provider.preparation.status()).toEqual({ phase: 'ready' })
})
it('retries interruptions on its own within one prepare', async () => {
const provider = await setup(
{ 'model.int8.onnx': { breakAfter: 30_000, breakRequests: 2, chunkDelayMs: 2 } },
{ download: { maxRetries: 3, progressIntervalMs: 0, probeTimeoutMs: 100 } },
)
const { states, error } = await prepareCollecting(provider)
expect(error).toBeUndefined()
expect(states.at(-1)).toEqual({ phase: 'ready' })
expect(states.some(state => (state.resumedFromBytes ?? 0) > 0)).toBe(true)
})
it('cancels, keeps the partial download, and continues it next time', async () => {
const provider = await setup({ 'model.int8.onnx': { chunkDelayMs: 5 } })
const controller = new AbortController()
const states: VoicePreparationState[] = []
const error = await provider.preparation.prepare(controller.signal, state => {
states.push(state)
if (state.resource === 'model.int8.onnx' && (state.completedBytes ?? 0) > assets!.items[0]!.asset.bytes + assets!.items[1]!.asset.bytes + 40_000) {
controller.abort()
}
}).catch(e => e)
expect(error).toBeDefined()
expect(states.at(-1)).toMatchObject({ phase: 'cancelled', totalBytes: assets!.totalBytes })
const layout = senseVoiceLayout(dataRoot)
expect(await exists(partPathFor(join(layout.models, 'model.int8.onnx')))).toBe(true)
const again = await prepareCollecting(provider)
expect(again.error).toBeUndefined()
expect(again.states.some(state => state.resumedFromBytes !== undefined)).toBe(true)
expect(await provider.preparation.status()).toEqual({ phase: 'ready' })
})
it('reports HTTP failures with the failing resource and status', async () => {
const provider = await setup({ 'silero_vad.onnx': { status: 404 } })
const { states, error } = await prepareCollecting(provider)
expect(error).toBeDefined()
expect(states.at(-1)).toMatchObject({
phase: 'failed', step: 'vad', resource: 'silero_vad.onnx',
error: { reason: 'http', status: 404, resource: 'silero_vad.onnx' },
})
// Items that already verified stay installed.
expect(await exists(join(senseVoiceLayout(dataRoot).models, 'model.int8.onnx'))).toBe(true)
})
it('rejects a tampered download and installs nothing from it', async () => {
const tampered = Buffer.from(makeContent(120_000, 1))
tampered[10] ^= 1
const provider = await setup({}, {}, { 'model.int8.onnx': tampered })
const { states } = await prepareCollecting(provider)
expect(states.at(-1)).toMatchObject({ phase: 'failed', error: { reason: 'integrity' } })
const layout = senseVoiceLayout(dataRoot)
expect(await exists(join(layout.models, 'model.int8.onnx'))).toBe(false)
expect(await exists(partPathFor(join(layout.models, 'model.int8.onnx')))).toBe(false)
expect((await provider.preparation.status()).phase).toBe('unprepared')
})
it('fails when a runtime package does not contain its entry file', async () => {
const provider = await setup({}, {
extract: async ({ cwd, into }) => {
// Extracts nothing: an archive with an unexpected layout.
await writeFile(join(cwd, into, 'unrelated.txt'), 'x')
},
})
const { states } = await prepareCollecting(provider)
expect(states.at(-1)).toMatchObject({ phase: 'failed', step: 'runtime', error: { reason: 'integrity' } })
})
it('reports a failed unpack as a storage problem', async () => {
const provider = await setup({}, {
extract: async () => { throw Object.assign(new Error('tar exploded'), { code: 'ENOSPC' }) },
})
const { states } = await prepareCollecting(provider)
expect(states.at(-1)).toMatchObject({ phase: 'failed', error: { reason: 'storage' } })
})
it('refuses a second concurrent prepare', async () => {
const provider = await setup({ 'model.int8.onnx': { chunkDelayMs: 20 } })
const running = prepareCollecting(provider)
await new Promise(resolve => setTimeout(resolve, 30))
const second = await provider.preparation.prepare(new AbortController().signal, () => {}).catch(e => e)
expect(second.message).toContain('already running')
expect((await running).error).toBeUndefined()
})
})
describe('remove', () => {
it('deletes the runtime, models and partial files', async () => {
const provider = await setup()
await prepareCollecting(provider)
await provider.preparation.remove()
expect(await exists(senseVoiceLayout(dataRoot).base)).toBe(false)
expect(await provider.preparation.status()).toEqual({ phase: 'unprepared' })
})
it('stops a running worker before deleting', async () => {
const provider = await setup()
await prepareCollecting(provider)
await provider.transcribe(makeWav(1), { language: 'zh' }, new AbortController().signal)
const [pid] = (await readFile(join(senseVoiceLayout(dataRoot).models, 'starts.log'), 'utf8')).split('\n')
await provider.preparation.remove()
expect(() => process.kill(Number(pid), 0)).toThrow()
})
})
describe('transcribe', () => {
const signal = () => new AbortController().signal
it('refuses to run before the assets are installed', async () => {
const provider = await setup()
const error = await provider.transcribe(makeWav(1), { language: 'zh' }, signal()).catch(e => e)
expect(error).toBeInstanceOf(VoiceServiceError)
expect(error.code).toBe('voice/not-ready')
})
it('transcribes through the worker once installed', async () => {
const provider = await setup()
await prepareCollecting(provider)
const result = await provider.transcribe(makeWav(1), { language: 'ja' }, signal())
expect(result).toEqual({ text: 'echo:ja', audioSeconds: 1, inferenceSeconds: 0.01 })
})
it('reclaims the idle worker after the configured time', async () => {
const provider = await setup({}, { idleTimeoutMs: 100 })
await prepareCollecting(provider)
await provider.transcribe(makeWav(1), { language: 'zh' }, signal())
const [pid] = (await readFile(join(senseVoiceLayout(dataRoot).models, 'starts.log'), 'utf8')).split('\n')
const deadline = Date.now() + 3000
const alive = () => { try { process.kill(Number(pid), 0); return true } catch { return false } }
while (alive() && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 20))
expect(alive()).toBe(false)
})
})
@@ -0,0 +1,199 @@
/**
* Local SenseVoice Small (INT8) speech provider.
*
* The runtime (sherpa-onnx) and the model are downloaded into the data
* directory on demand, so the installer stays small. Recognition runs in a
* separate worker process that is started on demand and reclaimed when idle.
*/
import { join } from 'node:path'
import { getNetworkProxyFetchOptions, loadNetworkSettings } from '../../networkSettings.js'
import type { DownloadOptions, FetchLike } from '../download/index.js'
import { VoiceServiceError } from '../errors.js'
import {
VOICE_LIMITS,
type SpeechProvider,
type VoiceLanguage,
type VoicePreparationState,
type VoiceProviderInfo,
type VoiceTranscript,
} from '../types.js'
import {
installItems,
resolveRuntimePlatform,
totalDownloadBytes,
type InstallItem,
} from './assets.js'
import {
bytesOnDisk,
failureState,
installAll,
isInstalled,
removeInstall,
type ExtractArchive,
} from './install.js'
import { defaultVoiceDataRoot, senseVoiceLayout, type SenseVoiceLayout } from './layout.js'
import type { WorkerConfig } from './protocol.js'
import { SenseVoiceRecognizer, type SpawnWorker } from './recognizer.js'
export const SENSEVOICE_PROVIDER_ID = 'sensevoice-local'
export interface SenseVoiceProviderOptions {
/** Defaults to `<config dir>/cc-haha/voice`. */
dataRoot?: string
fetch?: FetchLike
/** Extra fetch options per URL (proxy). Defaults to the configured network proxy. */
fetchOptions?: DownloadOptions['fetchOptions']
sleep?: DownloadOptions['sleep']
/** Idle time before the worker is stopped. Default 300000. */
idleTimeoutMs?: number
spawnWorker?: SpawnWorker
/** Override for tests; defaults to the running process platform. */
platform?: { platform: NodeJS.Platform; arch: string }
/** Replaces the pinned asset list (tests use local fixtures). */
items?: InstallItem[]
extract?: ExtractArchive
threads?: number
startupTimeoutMs?: number
inferenceTimeoutMs?: number
/** Download tuning: retries, backoff, probe/idle timeouts, progress interval. */
download?: Pick<DownloadOptions, 'maxRetries' | 'backoffMs' | 'maxBackoffMs' | 'probeTimeoutMs' | 'idleTimeoutMs' | 'progressIntervalMs'>
}
export interface SenseVoiceProvider extends SpeechProvider {
readonly preparation: NonNullable<SpeechProvider['preparation']>
/** Stops the worker process and rejects new work. */
dispose(): Promise<void>
}
const LANGUAGES: VoiceLanguage[] = ['auto', 'zh', 'en', 'ja', 'ko', 'yue']
async function proxyFetchOptions(url: string): Promise<Record<string, unknown>> {
try {
return getNetworkProxyFetchOptions(await loadNetworkSettings(), url) as Record<string, unknown>
} catch {
// Proxy settings unavailable: fall back to a direct request.
return {}
}
}
export function createSenseVoiceProvider(options: SenseVoiceProviderOptions = {}): SenseVoiceProvider {
const dataRoot = options.dataRoot ?? defaultVoiceDataRoot()
const layout: SenseVoiceLayout = senseVoiceLayout(dataRoot)
const runtimePlatform = resolveRuntimePlatform(options.platform?.platform, options.platform?.arch)
const items = options.items ?? installItems(runtimePlatform)
const supported = options.items !== undefined || runtimePlatform !== undefined
const totalBytes = totalDownloadBytes(items)
const info: VoiceProviderInfo = {
id: SENSEVOICE_PROVIDER_ID,
name: 'SenseVoice Small (INT8)',
location: 'local',
languages: LANGUAGES,
downloadBytes: totalBytes,
}
const unsupportedState = (): VoicePreparationState => ({
phase: 'failed',
error: {
reason: 'unsupported-platform',
message: `Local speech recognition is not available on ${options.platform?.platform ?? process.platform}-${options.platform?.arch ?? process.arch}`,
},
})
const fetchOptions = options.fetchOptions ?? (options.fetch ? undefined : proxyFetchOptions)
const downloadOptions: DownloadOptions = {
...options.download,
...(options.fetch ? { fetch: options.fetch } : {}),
...(fetchOptions ? { fetchOptions } : {}),
...(options.sleep ? { sleep: options.sleep } : {}),
}
const modelPath = (name: string): string => join(layout.models, name)
const recognizer = new SenseVoiceRecognizer({
cwd: layout.base,
idleTimeoutMs: options.idleTimeoutMs,
startupTimeoutMs: options.startupTimeoutMs,
inferenceTimeoutMs: options.inferenceTimeoutMs,
spawnWorker: options.spawnWorker,
workerConfig: (): WorkerConfig => ({
runtimeRoot: layout.runtimeRoot,
model: modelPath('model.int8.onnx'),
tokens: modelPath('tokens.txt'),
vad: modelPath('silero_vad.onnx'),
threads: options.threads ?? 2,
maxAudioBytes: VOICE_LIMITS.maxAudioBytes,
vadThreshold: 0.5,
minSilenceSeconds: 0.5,
minSpeechSeconds: 0.25,
segmentSeconds: 30,
}),
})
let preparing = false
const status = async (): Promise<VoicePreparationState> => {
if (!supported) return unsupportedState()
if (await isInstalled(layout, items)) return { phase: 'ready' }
const completedBytes = await bytesOnDisk(layout, items)
return completedBytes > 0
? { phase: 'unprepared', completedBytes, totalBytes }
: { phase: 'unprepared' }
}
const prepare: SenseVoiceProvider['preparation']['prepare'] = async (signal, report) => {
if (!supported) {
const state = unsupportedState()
report(state)
throw new VoiceServiceError('voice/failed', state.error!.message)
}
if (preparing) throw new Error('Speech preparation is already running')
preparing = true
let latest: VoicePreparationState = { phase: 'downloading' }
const track = (state: VoicePreparationState): void => {
latest = state
report(state)
}
try {
await installAll({ layout, items, download: downloadOptions, extract: options.extract }, signal, track)
report({ phase: 'ready' })
} catch (error) {
if (signal.aborted) {
report({
phase: 'cancelled',
...(latest.completedBytes !== undefined ? { completedBytes: latest.completedBytes } : {}),
totalBytes,
})
throw error
}
const state = failureState(error, {
step: latest.step,
resource: latest.resource,
completedBytes: latest.completedBytes,
totalBytes,
})
report(state)
throw error
} finally {
preparing = false
}
}
return {
info,
preparation: {
status,
prepare,
async remove() {
await recognizer.stopWorker()
await removeInstall(layout)
},
},
async transcribe(wav, { language }, signal): Promise<VoiceTranscript> {
if (!supported || !await isInstalled(layout, items)) {
throw new VoiceServiceError('voice/not-ready', 'Speech model is not downloaded yet')
}
return recognizer.transcribe(wav, language, signal)
},
dispose: () => recognizer.dispose(),
}
}
@@ -0,0 +1,272 @@
import { afterEach, beforeEach, describe, expect, it } from 'bun:test'
import { spawn } from 'node:child_process'
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { VoiceServiceError } from '../errors.js'
import type { WorkerConfig } from './protocol.js'
import { installFakeSherpa } from './__fixtures__/fakeSherpa.js'
import { makeWav } from './__fixtures__/wav.js'
import { SenseVoiceRecognizer, isBundledWorkerHost, type RecognizerOptions, type SpawnWorker } from './recognizer.js'
const fakeWorker = fileURLToPath(new URL('./__fixtures__/fakeWorker.mjs', import.meta.url))
const wav = new Uint8Array(64)
let dir: string
let recognizers: SenseVoiceRecognizer[]
const spawnFake: SpawnWorker = ({ env, cwd }) => spawn(process.execPath, [fakeWorker], { env, cwd, stdio: 'pipe' })
function config(): WorkerConfig {
return {
runtimeRoot: dir,
model: join(dir, 'model.onnx'),
tokens: join(dir, 'tokens.txt'),
vad: join(dir, 'vad.onnx'),
threads: 1,
maxAudioBytes: 1024,
vadThreshold: 0.5,
minSilenceSeconds: 0.5,
minSpeechSeconds: 0.25,
segmentSeconds: 30,
}
}
function create(overrides: Partial<RecognizerOptions> = {}): SenseVoiceRecognizer {
const recognizer = new SenseVoiceRecognizer({
workerConfig: config,
cwd: dir,
spawnWorker: spawnFake,
idleTimeoutMs: 0,
killGraceMs: 500,
...overrides,
})
recognizers.push(recognizer)
return recognizer
}
async function lines(name: string): Promise<string[]> {
const text = await readFile(join(dir, name), 'utf8').catch(() => '')
return text.split('\n').filter(Boolean)
}
function alive(pid: number): boolean {
try {
process.kill(pid, 0)
return true
} catch {
return false
}
}
async function until(predicate: () => boolean | Promise<boolean>, timeoutMs = 3000): Promise<void> {
const deadline = Date.now() + timeoutMs
while (!await predicate()) {
if (Date.now() > deadline) throw new Error('condition not met in time')
await new Promise(resolve => setTimeout(resolve, 10))
}
}
const signal = () => new AbortController().signal
beforeEach(async () => {
dir = await mkdtemp(join(tmpdir(), 'voice-recognizer-'))
recognizers = []
})
afterEach(async () => {
await Promise.all(recognizers.map(recognizer => recognizer.dispose()))
await rm(dir, { recursive: true, force: true })
})
describe('SenseVoiceRecognizer', () => {
it('starts the worker on first use and reuses it afterwards', async () => {
const recognizer = create()
expect(await lines('starts.log')).toEqual([])
expect(recognizer.running).toBe(false)
expect(await recognizer.transcribe(wav, 'zh', signal())).toEqual({ text: 'echo:zh', audioSeconds: 1, inferenceSeconds: 0.01 })
await recognizer.transcribe(wav, 'en', signal())
expect(await lines('starts.log')).toHaveLength(1)
expect((await lines('requests.log')).map(line => line.split(' ')[1])).toEqual(['zh', 'en'])
expect(recognizer.running).toBe(true)
})
it('authenticates with a per-process token the worker receives only through its environment', async () => {
const recognizer = create()
let seen: Record<string, string> = {}
const recognizerWithSpy = create({
spawnWorker: input => {
seen = input.env
return spawnFake(input)
},
})
await recognizer.transcribe(wav, 'zh', signal())
await recognizerWithSpy.transcribe(wav, 'zh', signal())
expect(seen.CC_HAHA_VOICE_WORKER_TOKEN).toMatch(/^[a-f0-9]{64}$/)
expect(Object.keys(seen)).not.toContain('HTTP_PROXY')
expect(JSON.parse(seen.CC_HAHA_VOICE_WORKER_CONFIG!).model).toBe(join(dir, 'model.onnx'))
})
it('stops an idle worker and starts a fresh one for the next request', async () => {
const recognizer = create({ idleTimeoutMs: 100 })
await recognizer.transcribe(wav, 'zh', signal())
const [first] = await lines('starts.log')
expect(alive(Number(first))).toBe(true)
await until(() => !alive(Number(first)))
expect(recognizer.running).toBe(false)
await recognizer.transcribe(wav, 'zh', signal())
const starts = await lines('starts.log')
expect(starts).toHaveLength(2)
expect(starts[1]).not.toBe(first)
})
it('does not reclaim the worker while requests are queued', async () => {
const recognizer = create({ idleTimeoutMs: 100 })
const first = recognizer.transcribe(wav, 'slow', signal())
const second = recognizer.transcribe(wav, 'slow', signal())
await Promise.all([first, second])
expect(await lines('starts.log')).toHaveLength(1)
})
it('recovers after the worker crashes mid-request', async () => {
const recognizer = create()
await recognizer.transcribe(wav, 'zh', signal())
const crashed = await recognizer.transcribe(wav, 'crash', signal()).catch(error => error)
expect(crashed).toBeInstanceOf(VoiceServiceError)
expect(crashed.code).toBe('voice/failed')
expect(await recognizer.transcribe(wav, 'en', signal())).toMatchObject({ text: 'echo:en' })
expect(await lines('starts.log')).toHaveLength(2)
})
it('kills the worker when a request is cancelled and starts clean afterwards', async () => {
const recognizer = create()
const controller = new AbortController()
const pending = recognizer.transcribe(wav, 'hold', controller.signal)
await until(async () => (await lines('requests.log')).length === 1)
const [pid] = (await lines('requests.log'))[0]!.split(' ')
controller.abort(new Error('user cancelled'))
const error = await pending.catch(e => e)
expect(error.message).toBe('user cancelled')
await until(() => !alive(Number(pid)))
expect(await recognizer.transcribe(wav, 'zh', signal())).toMatchObject({ text: 'echo:zh' })
expect(await lines('starts.log')).toHaveLength(2)
})
it('kills a worker that exceeds the inference timeout', async () => {
const recognizer = create({ inferenceTimeoutMs: 150 })
const error = await recognizer.transcribe(wav, 'hold', signal()).catch(e => e)
expect(error).toBeInstanceOf(VoiceServiceError)
expect(error.code).toBe('voice/failed')
expect(error.message).toContain('timed out')
const [pid] = (await lines('requests.log'))[0]!.split(' ')
await until(() => !alive(Number(pid)))
expect(recognizer.running).toBe(false)
})
it('never sends a request that was cancelled while waiting in the queue', async () => {
const recognizer = create()
const running = recognizer.transcribe(wav, 'slow', signal())
const controller = new AbortController()
const waiting = recognizer.transcribe(wav, 'en', controller.signal)
controller.abort(new Error('changed my mind'))
expect((await waiting.catch(e => e)).message).toBe('changed my mind')
await running
expect((await lines('requests.log')).map(line => line.split(' ')[1])).toEqual(['slow'])
})
it('rejects when too many requests are pending', async () => {
const recognizer = create({ maxPending: 2 })
const results = [
recognizer.transcribe(wav, 'slow', signal()),
recognizer.transcribe(wav, 'slow', signal()),
]
const overflow = await recognizer.transcribe(wav, 'slow', signal()).catch(e => e)
expect(overflow).toBeInstanceOf(VoiceServiceError)
expect(overflow.message).toContain('queue is full')
await expect(Promise.all(results)).resolves.toHaveLength(2)
})
it('maps worker input rejections to invalid-audio and keeps the worker', async () => {
const recognizer = create()
const error = await recognizer.transcribe(wav, 'invalid-input', signal()).catch(e => e)
expect(error).toBeInstanceOf(VoiceServiceError)
expect(error.code).toBe('voice/invalid-audio')
await recognizer.transcribe(wav, 'zh', signal())
expect(await lines('starts.log')).toHaveLength(1)
})
it('treats worker errors and malformed output as failures and restarts the worker', async () => {
const recognizer = create()
for (const language of ['error', 'garbage']) {
const error = await recognizer.transcribe(wav, language, signal()).catch(e => e)
expect(error.code).toBe('voice/failed')
}
await recognizer.transcribe(wav, 'zh', signal())
expect(await lines('starts.log')).toHaveLength(3)
})
it('reports a startup failure with the worker diagnostics, then starts once the cause is gone', async () => {
const recognizer = create()
await writeFile(join(dir, 'fail-start'), '')
const error = await recognizer.transcribe(wav, 'zh', signal()).catch(e => e)
expect(error.code).toBe('voice/failed')
expect(error.message).toContain('model failed to load')
await rm(join(dir, 'fail-start'))
expect(await recognizer.transcribe(wav, 'zh', signal())).toMatchObject({ text: 'echo:zh' })
})
it('gives up when the worker does not become ready in time', async () => {
const recognizer = create({ startupTimeoutMs: 50 })
await writeFile(join(dir, 'slow-start'), '')
const error = await recognizer.transcribe(wav, 'zh', signal()).catch(e => e)
expect(error.code).toBe('voice/failed')
expect(error.message).toContain('startup timed out')
const [pid] = await lines('starts.log')
await until(() => !alive(Number(pid)))
})
it('stops the worker and refuses new work after dispose', async () => {
const recognizer = create()
await recognizer.transcribe(wav, 'zh', signal())
const [pid] = await lines('starts.log')
await recognizer.dispose()
expect(alive(Number(pid))).toBe(false)
await expect(recognizer.transcribe(wav, 'zh', signal())).rejects.toThrow('disposed')
})
})
describe('default worker launch', () => {
it('runs the source worker entry with Bun when not compiled, passing config and token through the environment', async () => {
await installFakeSherpa(dir)
// No spawnWorker override: this is the launch path the server uses from source.
const recognizer = create({
spawnWorker: undefined,
workerConfig: () => ({ ...config(), maxAudioBytes: 200_000 }),
})
const result = await recognizer.transcribe(makeWav(1), 'ko', signal())
expect(result).toMatchObject({ text: 'ko:16000', audioSeconds: 1 })
})
it('detects compiled executables by their virtual module URLs', () => {
expect(isBundledWorkerHost('file:///$bunfs/root/claude-sidecar', false)).toBe(true)
expect(isBundledWorkerHost('file:///B:/~BUN/root/claude-sidecar.exe', false)).toBe(true)
expect(isBundledWorkerHost('file:///repo/src/server/services/voice/sensevoice/recognizer.ts', false)).toBe(false)
expect(isBundledWorkerHost('file:///repo/src/x.ts', true)).toBe(true)
})
})
@@ -0,0 +1,383 @@
/**
* Server-side owner of the SenseVoice worker process: starts it on demand,
* serializes requests, recycles it when idle, and kills it whenever a request
* is cancelled, times out or the worker dies so the next request starts clean.
*/
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
import { randomBytes } from 'node:crypto'
import { realpath } from 'node:fs/promises'
import { request as httpRequest } from 'node:http'
import { fileURLToPath } from 'node:url'
import { isInBundledMode } from '../../../../utils/bundledMode.js'
import { VoiceServiceError } from '../errors.js'
import type { VoiceTranscript } from '../types.js'
import {
WORKER_CONFIG_ENV,
WORKER_TOKEN_ENV,
type WorkerConfig,
type WorkerErrorBody,
type WorkerTranscript,
} from './protocol.js'
export interface SpawnWorkerInput {
env: Record<string, string>
cwd: string
}
export type SpawnWorker = (input: SpawnWorkerInput) => ChildProcessWithoutNullStreams | Promise<ChildProcessWithoutNullStreams>
export interface RecognizerOptions {
/** Resolved lazily at spawn time so a freshly installed runtime is picked up. */
workerConfig: () => WorkerConfig | Promise<WorkerConfig>
cwd: string
spawnWorker?: SpawnWorker
/** Idle time before the worker is stopped; 0 keeps it warm. Default 300000. */
idleTimeoutMs?: number
/** Time allowed for a cold start including model load. Default 60000. */
startupTimeoutMs?: number
/** Time allowed for one inference. Default 120000. */
inferenceTimeoutMs?: number
/** Running plus waiting requests. Default 4. */
maxPending?: number
/** Wait between SIGTERM and SIGKILL. Default 1000. */
killGraceMs?: number
}
interface RunningWorker {
child: ChildProcessWithoutNullStreams
port: number
token: string
closed: boolean
stderr: string
}
const READY_LIMIT_BYTES = 4096
const RESPONSE_LIMIT_BYTES = 128 * 1024
/**
* True inside a `bun build --compile` executable, whose modules live in a virtual
* file system (`/$bunfs/...` on POSIX, `B:/~BUN/...` on Windows).
*/
export function isBundledWorkerHost(moduleUrl: string = import.meta.url, bundledMode: boolean = isInBundledMode()): boolean {
return bundledMode || moduleUrl.startsWith('file:///$bunfs/') || /^file:\/\/\/[A-Za-z]:\/~BUN\//.test(moduleUrl)
}
/**
* Launch command for the worker. A compiled desktop binary re-runs itself with
* `--voice-worker`; from source we run the worker entry with the current Bun.
*/
export async function defaultSpawnWorker({ env, cwd }: SpawnWorkerInput): Promise<ChildProcessWithoutNullStreams> {
const executable = await realpath(process.execPath)
const args = isBundledWorkerHost()
? ['--voice-worker']
: ['--no-env-file', fileURLToPath(new URL('./workerMain.ts', import.meta.url))]
return spawn(executable, args, {
cwd,
env,
stdio: 'pipe',
windowsHide: true,
})
}
/** Minimal environment for the worker: no inherited secrets, no proxy variables. */
export function workerEnvironment(extra: Record<string, string>): Record<string, string> {
const keep = ['PATH', 'HOME', 'USERPROFILE', 'TMPDIR', 'TMP', 'TEMP', 'SystemRoot', 'SYSTEMROOT', 'LANG', 'LC_ALL', 'LD_LIBRARY_PATH']
const env: Record<string, string> = {}
for (const key of keep) {
const value = process.env[key]
if (value) env[key] = value
}
return { ...env, BUN_OPTIONS: '--no-env-file', ...extra }
}
function abortReason(signal: AbortSignal): Error {
return signal.reason instanceof Error ? signal.reason : new Error('Speech recognition cancelled')
}
/** Resolves with the promise result, or rejects as soon as the signal aborts. */
function raceAbort<T>(pending: Promise<T>, signal: AbortSignal): Promise<T> {
if (signal.aborted) return Promise.reject(abortReason(signal))
return new Promise<T>((resolve, reject) => {
const onAbort = (): void => reject(abortReason(signal))
signal.addEventListener('abort', onAbort, { once: true })
pending.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort))
})
}
class WorkerCallError extends Error {
constructor(message: string, readonly invalidInput = false) {
super(message)
}
}
function postWav(
worker: RunningWorker,
wav: Uint8Array,
language: string,
signal: AbortSignal,
): Promise<WorkerTranscript> {
return new Promise((resolve, reject) => {
const req = httpRequest({
host: '127.0.0.1',
port: worker.port,
method: 'POST',
path: `/transcribe?language=${encodeURIComponent(language)}`,
headers: {
authorization: `Bearer ${worker.token}`,
'content-type': 'audio/wav',
'content-length': String(wav.byteLength),
},
// Loopback only: never route through an environment proxy.
agent: false,
}, response => {
const chunks: Buffer[] = []
let length = 0
response.on('data', (chunk: Buffer) => {
length += chunk.length
if (length > RESPONSE_LIMIT_BYTES) {
response.destroy(new WorkerCallError('Speech worker response exceeded its size limit'))
return
}
chunks.push(chunk)
})
response.once('error', reject)
response.once('end', () => {
let body: unknown
try {
body = JSON.parse(Buffer.concat(chunks).toString('utf8'))
} catch {
reject(new WorkerCallError('Speech worker returned malformed output'))
return
}
if (response.statusCode === 200) {
const value = body as Partial<WorkerTranscript>
if (typeof value.text === 'string' && typeof value.audioSeconds === 'number' && typeof value.inferenceSeconds === 'number') {
resolve({ text: value.text, audioSeconds: value.audioSeconds, inferenceSeconds: value.inferenceSeconds })
} else {
reject(new WorkerCallError('Speech worker returned an unexpected response'))
}
return
}
const failure = body as Partial<WorkerErrorBody>
reject(new WorkerCallError(
failure.error ?? `Speech worker responded ${response.statusCode}`,
failure.code === 'invalid-input' && (response.statusCode === 400 || response.statusCode === 413),
))
})
})
// Bun's http.request ignores the `signal` option, so cancellation is wired by hand.
// A destroyed request emits only 'close'; settle it there (a no-op if already settled).
const onAbort = (): void => {
req.destroy()
}
const cleanup = (): void => signal.removeEventListener('abort', onAbort)
req.once('error', error => {
cleanup()
reject(signal.aborted ? abortReason(signal) : error)
})
req.once('close', () => {
cleanup()
setImmediate(() => reject(signal.aborted ? abortReason(signal) : new WorkerCallError('Speech worker closed the connection')))
})
if (signal.aborted) {
req.destroy()
} else {
signal.addEventListener('abort', onAbort, { once: true })
req.end(Buffer.from(wav.buffer, wav.byteOffset, wav.byteLength))
}
})
}
export class SenseVoiceRecognizer {
private worker: RunningWorker | undefined
private tail: Promise<void> = Promise.resolve()
private pending = 0
private idle: ReturnType<typeof setTimeout> | undefined
private readonly lifetime = new AbortController()
private readonly idleTimeoutMs: number
private readonly startupTimeoutMs: number
private readonly inferenceTimeoutMs: number
private readonly maxPending: number
private readonly killGraceMs: number
private readonly spawnWorker: SpawnWorker
constructor(private readonly options: RecognizerOptions) {
this.idleTimeoutMs = options.idleTimeoutMs ?? 300_000
this.startupTimeoutMs = options.startupTimeoutMs ?? 60_000
this.inferenceTimeoutMs = options.inferenceTimeoutMs ?? 120_000
this.maxPending = options.maxPending ?? 4
this.killGraceMs = options.killGraceMs ?? 1000
this.spawnWorker = options.spawnWorker ?? defaultSpawnWorker
}
/** True while a worker process is alive. */
get running(): boolean {
return this.worker !== undefined && !this.worker.closed
}
transcribe(wav: Uint8Array, language: string, signal: AbortSignal): Promise<VoiceTranscript> {
return this.enqueue(async combined => this.execute(wav, language, combined), signal)
}
private enqueue<T>(run: (signal: AbortSignal) => Promise<T>, signal: AbortSignal): Promise<T> {
const combined = AbortSignal.any([signal, this.lifetime.signal])
if (combined.aborted) return Promise.reject(abortReason(combined))
if (this.pending >= this.maxPending) {
return Promise.reject(new VoiceServiceError('voice/failed', 'Speech transcription queue is full'))
}
clearTimeout(this.idle)
this.pending++
const job = this.tail.then(async () => {
combined.throwIfAborted()
return await run(combined)
})
this.tail = job.then(() => undefined, () => undefined).finally(() => {
this.pending--
if (this.pending === 0 && !this.lifetime.signal.aborted && this.idleTimeoutMs > 0) {
this.idle = setTimeout(() => {
this.tail = this.tail.then(async () => {
if (this.pending === 0) await this.stopWorker()
})
}, this.idleTimeoutMs)
this.idle.unref?.()
}
})
return job
}
private async execute(wav: Uint8Array, language: string, signal: AbortSignal): Promise<VoiceTranscript> {
let worker: RunningWorker
try {
worker = await this.ensureWorker(signal)
} catch (error) {
await this.stopWorker()
throw this.toServiceError(error, signal, 'start')
}
const call = new AbortController()
const timer = setTimeout(() => call.abort(new Error('Speech recognition timed out')), this.inferenceTimeoutMs)
try {
const result = await postWav(worker, wav, language, AbortSignal.any([signal, call.signal]))
return result
} catch (error) {
if (error instanceof WorkerCallError && error.invalidInput) {
throw new VoiceServiceError('voice/invalid-audio', error.message)
}
// The worker's state is unknown after any other failure: start over next time.
await this.stopWorker()
throw this.toServiceError(error, signal, 'inference', worker.stderr)
} finally {
clearTimeout(timer)
}
}
private toServiceError(error: unknown, signal: AbortSignal, phase: 'start' | 'inference', stderr = ''): Error {
if (signal.aborted) return abortReason(signal)
if (error instanceof VoiceServiceError) return error
const detail = error instanceof Error ? error.message : String(error)
const tail = stderr.trim() ? ` (${stderr.trim().slice(-300)})` : ''
return new VoiceServiceError('voice/failed', `Speech recognizer ${phase === 'start' ? 'could not start' : 'failed'}: ${detail}${tail}`)
}
private async ensureWorker(signal: AbortSignal): Promise<RunningWorker> {
if (this.worker && !this.worker.closed) return this.worker
if (this.worker) await this.stopWorker()
const config = await this.options.workerConfig()
const token = randomBytes(32).toString('hex')
const child = await this.spawnWorker({
cwd: this.options.cwd,
env: workerEnvironment({
[WORKER_CONFIG_ENV]: JSON.stringify(config),
[WORKER_TOKEN_ENV]: token,
}),
})
const worker: RunningWorker = { child, port: 0, token, closed: false, stderr: '' }
this.worker = worker
child.stderr.setEncoding('utf8')
child.stderr.on('data', (chunk: string) => { worker.stderr = (worker.stderr + chunk).slice(-4096) })
child.stdin.on('error', () => {})
const exited = new Promise<never>((_, reject) => {
child.on('error', error => {
worker.closed = true
reject(error)
})
child.once('exit', (code, exitSignal) => {
worker.closed = true
reject(new Error(`worker exited (${exitSignal ?? code})`))
})
})
exited.catch(() => {})
const startup = new AbortController()
const timer = setTimeout(() => startup.abort(new Error('Speech recognizer startup timed out')), this.startupTimeoutMs)
try {
worker.port = await raceAbort(
Promise.race([readPort(child), exited]),
AbortSignal.any([signal, startup.signal]),
)
} catch (error) {
if (worker.stderr.trim()) {
throw new Error(`${error instanceof Error ? error.message : String(error)} ${worker.stderr.trim().slice(-300)}`)
}
throw error
} finally {
clearTimeout(timer)
}
child.stdout.resume()
return worker
}
/** Terminates the worker if one exists; the next request starts a fresh one. */
async stopWorker(): Promise<void> {
const worker = this.worker
if (!worker) return
this.worker = undefined
worker.closed = true
const child = worker.child
if (child.exitCode !== null || child.signalCode !== null) return
const gone = new Promise<void>(resolve => child.once('exit', () => resolve()))
child.kill('SIGTERM')
const force = setTimeout(() => child.kill('SIGKILL'), this.killGraceMs)
try {
await gone
} finally {
clearTimeout(force)
}
}
/** Stops accepting work, lets the running request settle and terminates the worker. */
async dispose(): Promise<void> {
clearTimeout(this.idle)
this.lifetime.abort(new Error('Speech recognizer disposed'))
await this.tail
await this.stopWorker()
}
}
function readPort(child: ChildProcessWithoutNullStreams): Promise<number> {
return new Promise((resolve, reject) => {
let text = ''
const onData = (chunk: Buffer): void => {
text += chunk.toString('utf8')
if (Buffer.byteLength(text) > READY_LIMIT_BYTES) {
child.stdout.off('data', onData)
reject(new Error('Speech worker readiness line is too long'))
return
}
const end = text.indexOf('\n')
if (end < 0) return
child.stdout.off('data', onData)
try {
const value = JSON.parse(text.slice(0, end)) as { port?: unknown }
if (typeof value.port === 'number' && Number.isInteger(value.port) && value.port > 0 && value.port < 65536) {
resolve(value.port)
} else {
reject(new Error('Speech worker reported an invalid port'))
}
} catch {
reject(new Error('Speech worker printed an invalid readiness line'))
}
}
child.stdout.on('data', onData)
})
}
@@ -0,0 +1,23 @@
import { describe, expect, it } from 'bun:test'
import { makeWav } from './__fixtures__/wav.js'
import { WavError, readSamples } from './wav.js'
describe('readSamples', () => {
it('converts the PCM of a canonical WAV to floats in [-1, 1)', () => {
const samples = readSamples(makeWav(0.01))
expect(samples).toHaveLength(160)
expect(samples[1]).toBeCloseTo((37 - 1000) / 32768, 6)
for (const sample of samples) expect(Math.abs(sample)).toBeLessThan(1)
})
it.each([
['too short', new Uint8Array(20)],
['not a WAV', new Uint8Array(100)],
['non-canonical header', makeWav(1, { extraChunk: true })],
['empty data', makeWav(0)],
['odd data length', makeWav(1, { dataSizeOverride: 31_999 })],
['declared size past the end', makeWav(1, { dataSizeOverride: 999_999 })],
])('rejects %s instead of reading outside the buffer', (_name, bytes) => {
expect(() => readSamples(bytes)).toThrow(WavError)
})
})
@@ -0,0 +1,34 @@
/**
* Reads PCM from the canonical 44-byte-header 16 kHz mono PCM16 WAV the
* desktop recorder produces. The API layer has already validated the format;
* this only guards the worker against reading outside the buffer.
* No project imports, so the worker process can load it in isolation.
*/
const HEADER_BYTES = 44
export class WavError extends Error {
constructor(message: string) {
super(message)
this.name = 'WavError'
}
}
function tag(bytes: Uint8Array, offset: number): string {
return String.fromCharCode(...bytes.subarray(offset, offset + 4))
}
/** Returns the samples as floats in [-1, 1). Throws WavError on a malformed container. */
export function readSamples(bytes: Uint8Array): Float32Array {
if (bytes.byteLength < HEADER_BYTES || tag(bytes, 0) !== 'RIFF' || tag(bytes, 8) !== 'WAVE' || tag(bytes, 36) !== 'data') {
throw new WavError('Audio is not a canonical 44-byte-header WAV file')
}
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
const dataBytes = view.getUint32(40, true)
if (dataBytes === 0 || dataBytes % 2 !== 0 || dataBytes > bytes.byteLength - HEADER_BYTES) {
throw new WavError('WAV data length is empty, odd or past the end of the file')
}
const samples = new Float32Array(dataBytes / 2)
for (let i = 0; i < samples.length; i++) samples[i] = view.getInt16(HEADER_BYTES + i * 2, true) / 32768
return samples
}
@@ -0,0 +1,200 @@
import { afterEach, beforeEach, describe, expect, it } from 'bun:test'
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { installFakeSherpa } from './__fixtures__/fakeSherpa.js'
import { makeWav } from './__fixtures__/wav.js'
import { WORKER_CONFIG_ENV, WORKER_TOKEN_ENV, type WorkerConfig } from './protocol.js'
import { createTranscriber, type Sherpa } from './worker.js'
const workerMain = fileURLToPath(new URL('./workerMain.ts', import.meta.url))
const token = 'a'.repeat(64)
function baseConfig(runtimeRoot: string): WorkerConfig {
return {
runtimeRoot,
model: 'model.int8.onnx',
tokens: 'tokens.txt',
vad: 'silero_vad.onnx',
threads: 1,
maxAudioBytes: 200_000,
vadThreshold: 0.5,
minSilenceSeconds: 0.5,
minSpeechSeconds: 0.25,
segmentSeconds: 30,
}
}
describe('createTranscriber', () => {
function fakeSherpa() {
const calls = { languages: [] as string[], resets: 0, frontArgs: [] as unknown[], decoded: [] as number[] }
const segments: Float32Array[] = []
let buffered: Float32Array[] = []
const sherpa: Sherpa = {
OfflineRecognizer: class {
private language = 'auto'
createStream() {
return { acceptWaveform: (audio: { samples: Float32Array }) => { calls.decoded.push(audio.samples.length) } }
}
setConfig(config: object) {
this.language = (config as { modelConfig: { senseVoice: { language: string } } }).modelConfig.senseVoice.language
calls.languages.push(this.language)
}
decode() {}
getResult() {
const length = calls.decoded.at(-1)
return { text: length === 0 ? ' ' : ` ${this.language}-${length} ` }
}
} as unknown as Sherpa['OfflineRecognizer'],
Vad: class {
acceptWaveform(samples: Float32Array) { buffered.push(Float32Array.from(samples)) }
isEmpty() { return segments.length === 0 }
front(external: false) {
calls.frontArgs.push(external)
return { samples: segments[0]! }
}
pop() { segments.shift() }
reset() { calls.resets++; buffered = [] }
flush() {
const total = buffered.reduce((sum, part) => sum + part.length, 0)
// Two segments (one silent) so joining and dropping empties is exercised.
segments.push(new Float32Array(total), new Float32Array(0))
}
} as unknown as Sherpa['Vad'],
}
return { sherpa, calls }
}
it('segments with VAD, decodes each segment, joins non-empty text and copies buffers out of native memory', () => {
const { sherpa, calls } = fakeSherpa()
const transcribe = createTranscriber(baseConfig('/unused'), sherpa)
const result = transcribe(makeWav(1), 'zh')
expect(result.text).toBe('zh-16000')
expect(result.audioSeconds).toBe(1)
expect(result.inferenceSeconds).toBeGreaterThanOrEqual(0)
expect(calls.frontArgs.length).toBeGreaterThan(0)
expect(calls.frontArgs.every(arg => arg === false)).toBe(true)
})
it('sets the language and resets VAD state for every recording', () => {
const { sherpa, calls } = fakeSherpa()
const transcribe = createTranscriber(baseConfig('/unused'), sherpa)
transcribe(makeWav(0.5), 'zh')
transcribe(makeWav(0.5), 'en')
expect(calls.languages).toEqual(['zh', 'en'])
expect(calls.resets).toBe(2)
})
it('rejects unsupported languages and malformed audio before touching the recognizer', () => {
const { sherpa, calls } = fakeSherpa()
const transcribe = createTranscriber(baseConfig('/unused'), sherpa)
expect(() => transcribe(makeWav(1), 'fr')).toThrow('Unsupported SenseVoice language')
expect(() => transcribe(new Uint8Array(100), 'zh')).toThrow('WAV')
expect(calls.languages).toEqual([])
})
})
describe('worker process', () => {
let dir: string
let child: ChildProcessWithoutNullStreams | undefined
beforeEach(async () => {
dir = await mkdtemp(join(tmpdir(), 'voice-worker-'))
child = undefined
})
afterEach(async () => {
child?.kill('SIGKILL')
await rm(dir, { recursive: true, force: true })
})
function start(config: WorkerConfig): Promise<{ port: number }> {
child = spawn(process.execPath, ['--no-env-file', workerMain], {
cwd: dir,
stdio: 'pipe',
env: {
PATH: process.env.PATH ?? '',
[WORKER_CONFIG_ENV]: JSON.stringify(config),
[WORKER_TOKEN_ENV]: token,
},
})
let stderr = ''
child.stderr.on('data', chunk => { stderr += String(chunk) })
return new Promise((resolve, reject) => {
let text = ''
child!.stdout.on('data', chunk => {
text += String(chunk)
if (text.includes('\n')) resolve(JSON.parse(text.split('\n')[0]!))
})
child!.once('exit', code => reject(new Error(`worker exited ${code}: ${stderr}`)))
})
}
const post = (port: number, body: Uint8Array, options: { language?: string; token?: string; path?: string } = {}) =>
fetch(`http://127.0.0.1:${port}${options.path ?? '/transcribe'}?language=${options.language ?? 'zh'}`, {
method: 'POST',
headers: { authorization: `Bearer ${options.token ?? token}`, 'content-type': 'audio/wav' },
body,
// Keep loopback traffic off any ambient proxy.
proxy: '',
} as RequestInit)
it('loads sherpa from the runtime directory by absolute path and transcribes over the private socket', async () => {
await installFakeSherpa(dir)
const { port } = await start(baseConfig(dir))
const response = await post(port, makeWav(1), { language: 'ja' })
expect(response.status).toBe(200)
expect(await response.json()).toMatchObject({ text: 'ja:16000', audioSeconds: 1 })
})
it('rejects unauthenticated requests, unknown endpoints and bad audio without dying', async () => {
await installFakeSherpa(dir)
const { port } = await start(baseConfig(dir))
expect((await post(port, makeWav(1), { token: 'b'.repeat(64) })).status).toBe(401)
expect((await post(port, makeWav(1), { path: '/other' })).status).toBe(404)
expect((await post(port, new Uint8Array(10))).status).toBe(413)
const badLanguage = await post(port, makeWav(1), { language: 'fr' })
expect(badLanguage.status).toBe(400)
expect(await badLanguage.json()).toMatchObject({ code: 'invalid-input' })
expect((await post(port, new Uint8Array(100))).status).toBe(400)
// Still serving.
expect((await post(port, makeWav(1))).status).toBe(200)
})
it('exits cleanly when its parent closes stdin', async () => {
await installFakeSherpa(dir)
await start(baseConfig(dir))
const exited = new Promise<number | null>(resolve => child!.once('exit', code => resolve(code)))
child!.stdin.end()
expect(await exited).toBe(0)
})
it('fails fast with a diagnostic when the runtime is missing', async () => {
const error = await start(baseConfig(dir)).catch(e => e)
expect(error.message).toContain('worker exited 1')
expect(error.message).toContain('sherpa-onnx')
})
it('refuses to start without a token', async () => {
child = spawn(process.execPath, ['--no-env-file', workerMain], {
cwd: dir,
stdio: 'pipe',
env: { PATH: process.env.PATH ?? '', [WORKER_CONFIG_ENV]: JSON.stringify(baseConfig(dir)) },
})
let stderr = ''
child.stderr.on('data', chunk => { stderr += String(chunk) })
const code = await new Promise<number | null>(resolve => child!.once('exit', c => resolve(c)))
expect(code).toBe(1)
expect(stderr).toContain('without its configuration')
})
})
@@ -0,0 +1,217 @@
/**
* SenseVoice worker process: loads sherpa-onnx, then serves transcriptions over
* an authenticated loopback HTTP socket. Native inference is synchronous, so
* requests are naturally serialized. Nothing here may import server modules; in
* the compiled desktop app this file is reached through `claude-sidecar --voice-worker`.
*/
import { timingSafeEqual } from 'node:crypto'
import { createServer, type Server } from 'node:http'
import { createRequire } from 'node:module'
import { join } from 'node:path'
import {
WORKER_CONFIG_ENV,
WORKER_LANGUAGES,
WORKER_TOKEN_ENV,
type WorkerConfig,
type WorkerErrorBody,
type WorkerTranscript,
} from './protocol.js'
import { WavError, readSamples } from './wav.js'
interface Stream {
acceptWaveform(audio: { samples: Float32Array; sampleRate: number }): void
}
interface Recognizer {
createStream(): Stream
setConfig(config: object): void
decode(stream: Stream): void
getResult(stream: Stream): { text: string }
}
interface Detector {
acceptWaveform(samples: Float32Array): void
isEmpty(): boolean
front(externalBuffer: false): { samples: Float32Array }
pop(): void
reset(): void
flush(): void
}
export interface Sherpa {
OfflineRecognizer: new (config: object) => Recognizer
Vad: new (config: object, bufferSeconds: number) => Detector
}
const VAD_WINDOW = 512
/**
* Loads sherpa-onnx-node from an explicit path. In a compiled Bun binary,
* resolving the package by name or directory fails; the absolute entry file works.
*/
export function loadSherpa(runtimeRoot: string): Sherpa {
const entry = join(runtimeRoot, 'node_modules', 'sherpa-onnx-node', 'sherpa-onnx.js')
return createRequire(join(runtimeRoot, 'package.json'))(entry) as Sherpa
}
class InvalidInput extends Error {}
/** Builds the synchronous WAV-to-text function around one loaded model and VAD. */
export function createTranscriber(
config: WorkerConfig,
sherpa: Sherpa,
): (wav: Uint8Array, language: string) => WorkerTranscript {
const nativeConfig = {
featConfig: { sampleRate: 16000, featureDim: 80 },
modelConfig: {
senseVoice: { model: config.model, language: 'auto', useInverseTextNormalization: 1 },
tokens: config.tokens,
numThreads: config.threads,
provider: 'cpu',
debug: 0,
},
}
const recognizer = new sherpa.OfflineRecognizer(nativeConfig)
const detector = new sherpa.Vad({
sileroVad: {
model: config.vad,
threshold: config.vadThreshold,
minSilenceDuration: config.minSilenceSeconds,
minSpeechDuration: config.minSpeechSeconds,
maxSpeechDuration: config.segmentSeconds,
windowSize: VAD_WINDOW,
},
sampleRate: 16000,
numThreads: config.threads,
provider: 'cpu',
debug: 0,
}, config.segmentSeconds + config.minSilenceSeconds + 1)
return (wav, language) => {
if (!WORKER_LANGUAGES.includes(language)) throw new InvalidInput('Unsupported SenseVoice language')
let samples: Float32Array
try {
samples = readSamples(wav)
} catch (error) {
if (error instanceof WavError) throw new InvalidInput(error.message)
throw error
}
nativeConfig.modelConfig.senseVoice.language = language
recognizer.setConfig(nativeConfig)
detector.reset()
const started = performance.now()
const texts: string[] = []
const drain = (): void => {
while (!detector.isEmpty()) {
// Copy the segment out of native memory; the compiled runtime forbids external buffers.
const segment = detector.front(false)
const stream = recognizer.createStream()
stream.acceptWaveform({ sampleRate: 16000, samples: segment.samples })
recognizer.decode(stream)
texts.push(recognizer.getResult(stream).text.trim())
detector.pop()
}
}
for (let offset = 0; offset < samples.length; offset += VAD_WINDOW) {
detector.acceptWaveform(samples.subarray(offset, offset + VAD_WINDOW))
drain()
}
detector.flush()
drain()
return {
text: texts.filter(Boolean).join(' ').trim(),
audioSeconds: samples.length / 16000,
inferenceSeconds: (performance.now() - started) / 1000,
}
}
}
function reply(response: import('node:http').ServerResponse, status: number, body: WorkerTranscript | WorkerErrorBody): void {
response.writeHead(status, { 'content-type': 'application/json' }).end(JSON.stringify(body))
}
/** Binds an ephemeral loopback listener. Model loading has already finished. */
export async function startWorkerServer(
token: string,
maxAudioBytes: number,
transcribe: (wav: Uint8Array, language: string) => WorkerTranscript,
): Promise<{ server: Server; port: number }> {
const expected = Buffer.from(`Bearer ${token}`)
const server = createServer((request, response) => {
const supplied = Buffer.from(request.headers.authorization ?? '')
if (supplied.length !== expected.length || !timingSafeEqual(supplied, expected)) {
request.resume()
reply(response, 401, { error: 'Unauthorized' })
return
}
const url = new URL(request.url ?? '/', 'http://localhost')
if (request.method !== 'POST' || url.pathname !== '/transcribe') {
request.resume()
reply(response, 404, { error: 'Unknown endpoint' })
return
}
const length = Number(request.headers['content-length'])
if (!Number.isSafeInteger(length) || length < 46 || length > maxAudioBytes) {
request.resume()
reply(response, 413, { error: 'Invalid speech audio size', code: 'invalid-input' })
return
}
void (async () => {
try {
const chunks: Buffer[] = []
for await (const chunk of request) chunks.push(chunk as Buffer)
reply(response, 200, transcribe(Buffer.concat(chunks), url.searchParams.get('language') ?? 'auto'))
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
reply(response, error instanceof InvalidInput ? 400 : 500, {
error: message,
...(error instanceof InvalidInput ? { code: 'invalid-input' as const } : {}),
})
}
})()
})
await new Promise<void>((resolve, reject) => {
server.once('error', reject)
server.listen(0, '127.0.0.1', () => {
server.off('error', reject)
resolve()
})
})
return { server, port: (server.address() as { port: number }).port }
}
function readConfig(env: NodeJS.ProcessEnv): { config: WorkerConfig; token: string } {
const raw = env[WORKER_CONFIG_ENV]
const token = env[WORKER_TOKEN_ENV]
delete env[WORKER_CONFIG_ENV]
delete env[WORKER_TOKEN_ENV]
if (!raw || !token || !/^[A-Za-z0-9_-]{32,}$/.test(token)) {
throw new Error('Voice worker was started without its configuration')
}
const config = JSON.parse(raw) as WorkerConfig
for (const key of ['runtimeRoot', 'model', 'tokens', 'vad'] as const) {
if (typeof config[key] !== 'string' || !config[key]) throw new Error(`Voice worker config is missing ${key}`)
}
return { config, token }
}
/**
* Worker entry. Resolves only when the parent closes stdin or sends SIGTERM/SIGINT,
* so the process also ends if the server that spawned it disappears.
*/
export async function runVoiceWorker(env: NodeJS.ProcessEnv = process.env): Promise<void> {
const { config, token } = readConfig(env)
const transcribe = createTranscriber(config, loadSherpa(config.runtimeRoot))
const { server, port } = await startWorkerServer(token, config.maxAudioBytes, transcribe)
const finished = new Promise<void>(resolve => {
const stop = (): void => resolve()
process.stdin.once('end', stop)
process.stdin.once('close', stop)
process.once('SIGTERM', stop)
process.once('SIGINT', stop)
})
process.stdin.resume()
process.stdout.write(`${JSON.stringify({ port })}\n`)
await finished
server.close()
}
@@ -0,0 +1,10 @@
/** Development entry for the SenseVoice worker; packaged builds use `claude-sidecar --voice-worker`. */
import { runVoiceWorker } from './worker.js'
try {
await runVoiceWorker()
process.exit(0)
} catch (error) {
process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`)
process.exit(1)
}
+125
View File
@@ -0,0 +1,125 @@
/**
* Voice input (dictation) shared contract.
*
* The desktop renderer records audio, uploads a 16 kHz mono PCM16 WAV, and the
* server transcribes it through a registered SpeechProvider. `desktop/src/api/voice.ts`
* mirrors the wire types below; keep the two in sync (a parity test pins them).
*/
/** Language hint codes accepted by providers. `auto` lets the model detect. */
export type VoiceLanguage = 'auto' | 'zh' | 'en' | 'ja' | 'ko' | 'yue'
export type VoicePreparationPhase = 'unprepared' | 'downloading' | 'verifying' | 'ready' | 'failed' | 'cancelled'
/** Which resource is currently being fetched or checked. */
export type VoicePreparationStep = 'runtime' | 'model' | 'vad' | 'verify'
export type VoiceFailureReason =
| 'network'
| 'dns'
| 'timeout'
| 'certificate'
| 'http'
| 'integrity'
| 'storage'
| 'unsupported-platform'
| 'unknown'
export interface VoiceFailure {
reason: VoiceFailureReason
/** Origin that failed, credentials stripped. */
source?: string
/** HTTP status when reason is `http`. */
status?: number
/** Resource file name, e.g. `model.int8.onnx`. */
resource?: string
message: string
}
export interface VoicePreparationState {
phase: VoicePreparationPhase
step?: VoicePreparationStep
resource?: string
completedBytes?: number
totalBytes?: number
/** Bytes already on disk when this download resumed after an interruption. */
resumedFromBytes?: number
/** Which mirror/origin is serving the current download. */
source?: string
error?: VoiceFailure
}
export interface VoiceProviderInfo {
id: string
name: string
/** `local` runs on the machine hosting the server; `cloud` uploads audio. */
location: 'local' | 'cloud'
languages: VoiceLanguage[]
/** Total download size in bytes for local providers (runtime + models). */
downloadBytes?: number
}
export interface VoiceProviderStatus {
info: VoiceProviderInfo
preparation: VoicePreparationState
}
export interface VoicePreferences {
enabled: boolean
providerId: string
language: VoiceLanguage
}
export interface VoiceLimits {
maxAudioSeconds: number
maxAudioBytes: number
}
export interface VoiceCatalog {
/** False on platforms without a local runtime build; UI hides the feature. */
supported: boolean
providers: VoiceProviderStatus[]
preferences: VoicePreferences
limits: VoiceLimits
}
export interface VoiceTranscript {
text: string
audioSeconds: number
inferenceSeconds: number
}
export type VoiceErrorCode = 'voice/invalid-audio' | 'voice/not-ready' | 'voice/failed' | 'voice/unknown-provider'
export interface VoiceErrorBody {
error: VoiceErrorCode
message: string
}
export const VOICE_LIMITS: VoiceLimits = {
maxAudioSeconds: 120,
// 120 s of 16 kHz mono PCM16 is 3,840,044 bytes.
maxAudioBytes: 4 * 1024 * 1024,
}
export const DEFAULT_VOICE_PREFERENCES: VoicePreferences = {
enabled: false,
providerId: 'sensevoice-local',
language: 'auto',
}
/** Minimal provider contract. A registry is a plain Map keyed by `info.id`. */
export interface SpeechProvider {
readonly info: VoiceProviderInfo
/** Present for providers that need local assets before first use. */
readonly preparation?: {
status(): Promise<VoicePreparationState>
prepare(signal: AbortSignal, report: (state: VoicePreparationState) => void): Promise<void>
remove(): Promise<void>
}
transcribe(
wav: Uint8Array,
options: { language: VoiceLanguage },
signal: AbortSignal,
): Promise<VoiceTranscript>
}
+270
View File
@@ -0,0 +1,270 @@
import { ApiError } from '../../middleware/errorHandler.js'
import { VoiceServiceError } from './errors.js'
import { isVoiceLanguage } from './preferences.js'
import type { VoicePreferencesStore } from './preferencesStore.js'
import type { VoiceProviderRegistry } from './registry.js'
import {
VOICE_LIMITS,
type SpeechProvider,
type VoiceCatalog,
type VoiceLanguage,
type VoicePreferences,
type VoicePreparationState,
type VoiceProviderStatus,
type VoiceTranscript,
} from './types.js'
import { validateVoiceWav } from './wav.js'
export type VoiceServiceOptions = {
registry: VoiceProviderRegistry
preferences: VoicePreferencesStore
}
type PrepareTask = {
controller: AbortController
state: VoicePreparationState
/** Settles once the provider's prepare() has fully stopped, even after cancel. */
settled: Promise<void>
}
function isUnsupportedPlatform(state: VoicePreparationState): boolean {
return state.phase === 'failed' && state.error?.reason === 'unsupported-platform'
}
function describeError(error: unknown): string {
return error instanceof Error ? error.message : String(error)
}
function providerLanguage(provider: SpeechProvider, preferred: VoiceLanguage): VoiceLanguage {
const { languages } = provider.info
if (languages.includes(preferred)) return preferred
return languages.includes('auto') ? 'auto' : (languages[0] ?? 'auto')
}
export class VoiceService {
private readonly registry: VoiceProviderRegistry
private readonly preferences: VoicePreferencesStore
private readonly tasks = new Map<string, PrepareTask>()
/** Last failed/cancelled outcome per provider; disk state alone would read as `unprepared`. */
private readonly outcomes = new Map<string, VoicePreparationState>()
/** Cancelled tasks whose provider.prepare() has not stopped yet. */
private readonly winding = new Map<string, Promise<void>>()
constructor(options: VoiceServiceOptions) {
this.registry = options.registry
this.preferences = options.preferences
}
private requireProvider(id: string): SpeechProvider {
const provider = this.registry.get(id)
if (!provider) {
throw new VoiceServiceError('voice/unknown-provider', `Unknown voice provider: ${id}`)
}
return provider
}
private async providerPreparation(provider: SpeechProvider): Promise<VoicePreparationState> {
const task = this.tasks.get(provider.info.id)
if (task) return task.state
if (!provider.preparation) return { phase: 'ready' }
const onDisk = await provider.preparation.status()
if (onDisk.phase === 'ready') {
this.outcomes.delete(provider.info.id)
return onDisk
}
return this.outcomes.get(provider.info.id) ?? onDisk
}
private async providerStatus(provider: SpeechProvider): Promise<VoiceProviderStatus> {
return { info: provider.info, preparation: await this.providerPreparation(provider) }
}
async catalog(): Promise<VoiceCatalog> {
const providers = await Promise.all(this.registry.list().map(provider => this.providerStatus(provider)))
return {
// Providers report `unsupported-platform` themselves; the feature is only
// unavailable when every one of them does.
supported: providers.some(item => !isUnsupportedPlatform(item.preparation)),
providers,
preferences: await this.preferences.read(),
limits: VOICE_LIMITS,
}
}
async status(providerId: string): Promise<VoiceProviderStatus> {
return this.providerStatus(this.requireProvider(providerId))
}
async updatePreferences(patch: unknown): Promise<VoicePreferences> {
if (!patch || typeof patch !== 'object' || Array.isArray(patch)) {
throw ApiError.badRequest('Voice preferences must be an object')
}
const input = patch as Record<string, unknown>
const next: Partial<VoicePreferences> = {}
if (input.enabled !== undefined) {
if (typeof input.enabled !== 'boolean') throw ApiError.badRequest('enabled must be a boolean')
next.enabled = input.enabled
}
if (input.providerId !== undefined) {
if (typeof input.providerId !== 'string' || !this.registry.get(input.providerId)) {
throw new VoiceServiceError('voice/unknown-provider', `Unknown voice provider: ${String(input.providerId)}`)
}
next.providerId = input.providerId
}
if (input.language !== undefined) {
if (!isVoiceLanguage(input.language)) throw ApiError.badRequest(`Unsupported language: ${String(input.language)}`)
next.language = input.language
}
const current = await this.preferences.read()
const provider = this.requireProvider(next.providerId ?? current.providerId)
if (next.language !== undefined) {
if (!provider.info.languages.includes(next.language)) {
throw ApiError.badRequest(`${provider.info.name} does not support language: ${next.language}`)
}
} else if (next.providerId !== undefined && !provider.info.languages.includes(current.language)) {
// Switching provider must not leave a language the new provider cannot serve.
next.language = providerLanguage(provider, current.language)
}
return this.preferences.update(next)
}
/** Starts (or joins) the background download for a provider; returns immediately. */
async prepare(providerId: string): Promise<VoiceProviderStatus> {
const provider = this.requireProvider(providerId)
const preparation = provider.preparation
if (!preparation || this.tasks.has(providerId)) {
return this.providerStatus(provider)
}
this.outcomes.delete(providerId)
const controller = new AbortController()
const task: PrepareTask = {
controller,
state: { phase: 'downloading' },
settled: Promise.resolve(),
}
this.tasks.set(providerId, task)
task.settled = this.runPrepare(provider, task)
return { info: provider.info, preparation: task.state }
}
private async runPrepare(provider: SpeechProvider, task: PrepareTask): Promise<void> {
const id = provider.info.id
const preparation = provider.preparation!
const isCurrent = () => this.tasks.get(id) === task
let final: VoicePreparationState
try {
await this.winding.get(id)
if (task.controller.signal.aborted) return
const onDisk = await preparation.status()
if (onDisk.phase === 'ready') {
final = onDisk
} else {
await preparation.prepare(task.controller.signal, state => {
if (isCurrent()) task.state = state
})
const after = await preparation.status()
if (after.phase === 'ready') {
final = after
} else if (task.state.phase === 'failed') {
final = task.state
} else {
final = {
phase: 'failed',
error: { reason: 'unknown', message: 'Preparation finished but the assets are not ready' },
}
}
}
} catch (error) {
if (task.controller.signal.aborted) {
final = { phase: 'cancelled' }
} else if (task.state.phase === 'failed') {
final = task.state
} else {
final = { phase: 'failed', error: { reason: 'unknown', message: describeError(error) } }
}
}
// A cancel or remove already retired this task; its outcome is recorded there.
if (!isCurrent()) return
this.tasks.delete(id)
if (final.phase === 'ready') this.outcomes.delete(id)
else this.outcomes.set(id, final)
}
async cancel(providerId: string): Promise<VoiceProviderStatus> {
const provider = this.requireProvider(providerId)
const task = this.tasks.get(providerId)
if (task) {
this.tasks.delete(providerId)
this.outcomes.set(providerId, { phase: 'cancelled' })
task.controller.abort()
// Keep a later prepare()/remove() from racing the winding-down download.
const winding: Promise<void> = task.settled.finally(() => {
if (this.winding.get(providerId) === winding) this.winding.delete(providerId)
})
this.winding.set(providerId, winding)
}
return this.providerStatus(provider)
}
async removeAssets(providerId: string): Promise<VoiceProviderStatus> {
const provider = this.requireProvider(providerId)
await this.cancel(providerId)
await this.winding.get(providerId)
this.outcomes.delete(providerId)
if (provider.preparation) {
const removing = provider.preparation.remove()
// A prepare() arriving mid-removal must wait for it; otherwise its fresh
// download would be deleted underneath it (or leave a half-removed tree).
const winding: Promise<void> = removing
.then(() => {}, () => {})
.finally(() => {
if (this.winding.get(providerId) === winding) this.winding.delete(providerId)
})
this.winding.set(providerId, winding)
try {
await removing
} catch (error) {
throw new VoiceServiceError('voice/failed', `Failed to remove voice assets: ${describeError(error)}`)
}
}
return this.providerStatus(provider)
}
async transcribe(
providerId: string | undefined,
wav: Uint8Array,
language: VoiceLanguage | undefined,
signal: AbortSignal,
): Promise<VoiceTranscript> {
const preferences = await this.preferences.read()
const provider = this.requireProvider(providerId ?? preferences.providerId)
if (language !== undefined && !provider.info.languages.includes(language)) {
throw ApiError.badRequest(`${provider.info.name} does not support language: ${language}`)
}
const effectiveLanguage = language ?? providerLanguage(provider, preferences.language)
const { audioSeconds } = validateVoiceWav(wav)
const preparation = await this.providerPreparation(provider)
if (preparation.phase !== 'ready') {
throw new VoiceServiceError('voice/not-ready', `${provider.info.name} is not ready (${preparation.phase})`)
}
try {
const transcript = await provider.transcribe(wav, { language: effectiveLanguage }, signal)
return { ...transcript, audioSeconds }
} catch (error) {
if (signal.aborted) throw error
// Providers classify their own failures (invalid audio, model removed
// after the readiness check); wrapping those would turn a 400/409 into 500.
if (error instanceof VoiceServiceError) throw error
throw new VoiceServiceError('voice/failed', `Transcription failed: ${describeError(error)}`)
}
}
}
+48
View File
@@ -0,0 +1,48 @@
import { VoiceServiceError } from './errors.js'
import { VOICE_LIMITS, type VoiceLimits } from './types.js'
const HEADER_BYTES = 44
const SAMPLE_RATE = 16_000
const BYTES_PER_SAMPLE = 2
const BYTES_PER_SECOND = SAMPLE_RATE * BYTES_PER_SAMPLE
function invalid(message: string): never {
throw new VoiceServiceError('voice/invalid-audio', message)
}
function tag(bytes: Uint8Array, offset: number): string {
return String.fromCharCode(bytes[offset]!, bytes[offset + 1]!, bytes[offset + 2]!, bytes[offset + 3]!)
}
/**
* Validates the canonical 44-byte-header WAV the desktop recorder produces:
* PCM, mono, 16 kHz, 16-bit, with a `data` chunk that runs to end of file.
* Returns the audio duration in seconds.
*/
export function validateVoiceWav(bytes: Uint8Array, limits: VoiceLimits = VOICE_LIMITS): { audioSeconds: number } {
if (bytes.byteLength > limits.maxAudioBytes) {
invalid(`Audio is too large (${bytes.byteLength} bytes, limit ${limits.maxAudioBytes})`)
}
if (bytes.byteLength < HEADER_BYTES) invalid('Audio is not a WAV file')
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
if (tag(bytes, 0) !== 'RIFF' || tag(bytes, 8) !== 'WAVE') invalid('Audio is not a WAV file')
if (tag(bytes, 12) !== 'fmt ' || view.getUint32(16, true) !== 16 || tag(bytes, 36) !== 'data') {
invalid('Unsupported WAV layout; expected a canonical 44-byte header')
}
if (view.getUint16(20, true) !== 1) invalid('Unsupported WAV encoding; expected PCM')
if (view.getUint16(22, true) !== 1) invalid('Unsupported WAV channel count; expected mono')
if (view.getUint32(24, true) !== SAMPLE_RATE) invalid(`Unsupported WAV sample rate; expected ${SAMPLE_RATE} Hz`)
if (view.getUint16(34, true) !== 16) invalid('Unsupported WAV bit depth; expected 16-bit')
const dataBytes = view.getUint32(40, true)
if (dataBytes !== bytes.byteLength - HEADER_BYTES) invalid('WAV data length does not match file length')
if (dataBytes === 0) invalid('Audio is empty')
if (dataBytes % BYTES_PER_SAMPLE !== 0) invalid('WAV data is not a whole number of samples')
const audioSeconds = dataBytes / BYTES_PER_SECOND
if (audioSeconds > limits.maxAudioSeconds) {
invalid(`Audio is too long (${audioSeconds.toFixed(1)} s, limit ${limits.maxAudioSeconds} s)`)
}
return { audioSeconds }
}