import { ref, onBeforeUnmount, watch } from "vue"; import { v4 as uuidv4 } from "uuid"; import RealTimeHuman from "@bddh/starling-realtime-client"; import { checkPlayUnMute } from "@/utils/audioUtils"; import { SilenceDetector } from "@/utils/SilenceDetector"; import { TokenManager } from "@/utils/tokenManager"; import type { ChatMessage, SubtitleMode } from "@/types"; const DEFAULT_TEXT = "你好!我是皮特教授,欢迎向我提问任何有关电池的问题。"; const OPENING_TTS_DELAY_MS = 200; const SILENCE_TIMEOUT_MS = 5000; // 最短有效语音时长(秒),低于此时长视为无效输入 const MIN_VALID_DURATION_SEC = 0.5; // 播报超时保护(毫秒):如果FINISHED回调未触发,自动恢复状态 const BROADCAST_TIMEOUT_MS = 60000; const APPID = "i-sfyrkdr0ubepa"; const APP_KEY = "72vpdansbpg8nxjbxzhp"; // 打断词列表:识别到这些词时自动打断播报 const INTERRUPT_KEYWORDS = ["稍等一下", "停一下", "抱歉打断一下", "不好意思", "对不起", "停", "闭嘴"]; /** * 检测文本是否包含实质内容(去除空白和标点后仍有字符) * 用于区分真实语音输入和环境噪音触发的空白ASR结果 */ const hasSubstantialContent = (text: string): boolean => { const cleaned = text.replace( /[\s\u3000.,!?;:,。!?;:、…—\-_·""''()()\[\]【】《》<>]/g, "", ); return cleaned.length > 0; }; /** * 检测文本是否包含打断词 */ const containsInterruptKeyword = (text: string): boolean => { const cleaned = text.replace( /[\s\u3000.,!?;:,。!?;:、…—\-_·""''()()\[\]【】《》<>]/g, "", ); return INTERRUPT_KEYWORDS.some((keyword) => cleaned.includes(keyword)); }; enum ReadyState { UNINSTANTIATED = -1, CONNECTING = 0, OPEN = 1, CLOSING = 2, CLOSED = 3, } /** * 拾音阶段状态机 * - idle: 空闲,可开始拾音 * - recording: 正在录音(BTN模式按住/RTC模式ASR确认有效语音) * - processing: 语音采集完成,ASR正在识别 * - waiting: 等待AI回复 * - broadcasting: 数字人正在播报,不可拾音 * * 合法状态转换: * idle → recording (BTN模式按下 / RTC模式ASR确认有效语音内容) * recording → processing (BTN松手 / RTC静音超时 / 收到ASR结果) * recording → idle (音频被中断) * processing → waiting (有效语音输入完成) * processing → idle (无效语音输入 / ASR错误 / 音频被中断) * waiting → broadcasting (AI回复到达,开始播报) * broadcasting → idle (播报完成 / 播报被打断 / 播报超时) * * 关键防护: * - idle状态下仅当ASR返回包含实质内容的QUERY时才转换到recording * - 环境噪音触发的空白/标点ASR结果在idle状态下被忽略 * - SilenceDetector声音回调不触发状态转换,仅用于静音超时检测 */ export type PickPhase = | "idle" | "recording" | "processing" | "waiting" | "broadcasting"; /** 合法状态转换映射表 */ const VALID_TRANSITIONS: Record = { idle: ["recording"], recording: ["processing", "idle"], processing: ["waiting", "idle"], waiting: ["broadcasting", "idle"], broadcasting: ["idle"], }; export function useDigitalHuman(options?: { onUserQueryComplete?: (text: string) => void; }) { const humanInstanceRef = ref(null); const realTimeVideoReady = ref(false); const wsConnected = ref(false); const videoIsMuted = ref(false); const checkOver = ref(false); const openingPlayed = ref(false); const audioMode = ref<"btn" | "rtc" | "">(""); const isRecording = ref(false); const canTransfer = ref(false); const isMuted = ref(true); const enableInterruptRef = ref(true); const pullAudioFromRTC = ref(false); const isConnected = ref(false); // ===== 拾音状态机 ===== const pickPhase = ref("idle"); const setPickPhase = (phase: PickPhase, reason: string) => { const prev = pickPhase.value; // 状态转换守卫:检查是否为合法转换 const allowed = VALID_TRANSITIONS[prev]; if (allowed && !allowed.includes(phase)) { console.warn( `[状态机] 非法状态转换: ${prev} → ${phase} (${reason}),已阻止`, ); return; } pickPhase.value = phase; console.info(`[状态机] pickPhase: ${prev} → ${phase} (${reason})`); }; // ===== 播报状态追踪 ===== // 数字人是否正在播报(TTS语音输出中) const isBroadcasting = ref(false); // 字幕相关 const subtitleMode = ref("ai-only"); const currentAISubtitle = ref(""); const currentUserSubtitle = ref(""); const chatMessages = ref([]); let streamingAIId: string | null = null; let streamingUserId: string | null = null; // Token 配置:使用 TokenManager 自动生成与刷新 const tokenManager = new TokenManager(APPID, APP_KEY); const token = ref(tokenManager.getToken()); // Token 更新时同步到 ref tokenManager.onTokenRefreshed((tokenInfo) => { token.value = tokenInfo.token; }); // 静音检测器(RTC模式专用) const silenceDetector = new SilenceDetector({ volumeThreshold: 8, silenceDurationMs: SILENCE_TIMEOUT_MS, }); // 录音时长追踪(BTN模式) let recordStartTime = 0; // 播报超时保护计时器 let broadcastTimeoutId: ReturnType | null = null; // 打断词检测时间戳:用于忽略打断后延迟到达的ASR结果 let lastInterruptTime = 0; const clearBroadcastTimeout = () => { if (broadcastTimeoutId !== null) { clearTimeout(broadcastTimeoutId); broadcastTimeoutId = null; } }; const startBroadcastTimeout = () => { clearBroadcastTimeout(); broadcastTimeoutId = setTimeout(() => { console.warn("[播报] 超时保护触发:FINISHED回调未收到,自动恢复状态"); isBroadcasting.value = false; if (pickPhase.value === "broadcasting") { setPickPhase("idle", "播报超时保护,恢复空闲"); if (audioMode.value === "rtc") { resumeAutoPickup(); } } }, BROADCAST_TIMEOUT_MS); }; /** * 进入播报状态 * RTC模式下保持麦克风开启,允许ASR识别打断词 */ const enterBroadcasting = () => { isBroadcasting.value = true; setPickPhase("broadcasting", "数字人开始播报"); startBroadcastTimeout(); // RTC模式:播报期间保持麦克风开启,允许ASR识别打断词 if (audioMode.value === "rtc") { humanInstanceRef.value?.muteMicrophone?.(false); isMuted.value = false; canTransfer.value = true; const resumed = silenceDetector.resume(); if (!resumed) { startSilenceDetection(); } console.info("[播报] RTC模式:麦克风保持开启,允许ASR识别打断词"); } console.info("[播报] 开始播报,等待FINISHED信号"); }; /** * 检测语音输入是否有效 */ const isValidVoiceInput = (text: string, durationSec: number): boolean => { const cleaned = text.replace( /[\s\u3000.,!?;:,。!?;:、…—\-_·""''()()\[\]【】《》<>]/g, "", ); if (!cleaned) { console.info("[语音有效性] 文本为空或仅含标点/空白,判定为无效输入"); return false; } if (durationSec < MIN_VALID_DURATION_SEC) { console.info( `[语音有效性] 录音时长 ${durationSec.toFixed(2)}s 不足 ${MIN_VALID_DURATION_SEC}s,判定为无效输入`, ); return false; } return true; }; /** * 无效语音输入处理:重置状态,恢复可拾音 */ const handleInvalidVoiceInput = () => { console.info("[语音有效性] 终止当前交互流程,不进入等待回复状态"); currentUserSubtitle.value = ""; // 先暂停静音检测器,防止恢复拾音时产生竞争 silenceDetector.pause(); setPickPhase("idle", "无效语音输入,恢复空闲"); if (audioMode.value === "rtc") { resumeAutoPickup(); } }; /** * 播报完成后的统一处理:恢复空闲状态,RTC模式下恢复拾音 */ const onBroadcastFinished = () => { clearBroadcastTimeout(); isBroadcasting.value = false; console.info("[播报] 数字人播报完成"); // 仅在 broadcasting 状态下恢复空闲,避免状态错乱 if (pickPhase.value === "broadcasting") { setPickPhase("idle", "播报完成,恢复空闲"); if (audioMode.value === "rtc") { resumeAutoPickup(); } } }; const addChatMessage = ( role: "user" | "ai", content: string, completed: boolean = false, ): ChatMessage => { const msg: ChatMessage = { id: uuidv4(), role, content, timestamp: Date.now(), completed, }; if (!completed) { if (role === "ai") { streamingAIId = msg.id; } else { streamingUserId = msg.id; } } chatMessages.value = [...chatMessages.value, msg]; return msg; }; const updateStreamingMessage = ( role: "user" | "ai", content: string, completed: boolean, ) => { const targetId = role === "ai" ? streamingAIId : streamingUserId; if (!targetId) return; const idx = chatMessages.value.findIndex((m) => m.id === targetId); if (idx === -1) return; const updated: ChatMessage = { ...chatMessages.value[idx], content, completed, }; chatMessages.value = [ ...chatMessages.value.slice(0, idx), updated, ...chatMessages.value.slice(idx + 1), ]; if (completed) { if (role === "ai") streamingAIId = null; else streamingUserId = null; } }; const startNewAIReply = () => { currentAISubtitle.value = ""; streamingAIId = null; }; const startNewUserQuery = () => { currentUserSubtitle.value = ""; streamingUserId = null; }; const handleDigitalHumanCallback = async (data: any) => { const { status, content } = data; console.info("[DH Callback]", status, content); if (status === "DH_LIB_MESSAGE") { switch (content.action) { case "DOWN_SUBTITLE": { const { content: subtitleContent, completed, type, } = JSON.parse(content.body); if (type === "QUERY") { // 播报期间检测打断词:识别到打断词后仅中断播报并恢复拾音,不调用LLM接口 if (pickPhase.value === "broadcasting") { const hasInterrupt = containsInterruptKeyword(subtitleContent); if (hasInterrupt) { console.info(`[打断词检测] 播报中识别到打断词: "${subtitleContent}",自动打断(仅中断播报,不调用LLM)`); interrupt(); } else { console.info(`[打断词检测] 播报中收到ASR结果: "${subtitleContent}",非打断词,忽略`); } break; } // waiting 状态下忽略ASR结果(正在等LLM回复,不应接受新输入) if (pickPhase.value === "waiting") { console.info("[状态机] 等待AI回复中,忽略QUERY字幕"); break; } // idle 状态下收到 QUERY:必须验证内容有实质才允许状态转换 // 环境噪音可能触发 ASR 产生空白/标点内容的 QUERY,不应进入识别 if (pickPhase.value === "idle") { // 忽略打断后3秒内到达的延迟ASR结果(避免打断词被当作新输入) if (Date.now() - lastInterruptTime < 3000) { console.info( "[状态机] 打断后延迟到达的ASR结果,忽略", ); break; } const hasSubstance = hasSubstantialContent(subtitleContent); if (!hasSubstance) { console.info( "[状态机] idle状态下QUERY内容无实质(空白/标点),忽略", ); break; } // 内容有实质:用户已开始有效说话 // 暂停静音检测器避免竞争,然后进入 recording 再到 processing if (audioMode.value === "rtc") { silenceDetector.pause(); } console.info( "[状态机] idle状态下收到有效ASR结果,用户已开始说话", ); setPickPhase("recording", "ASR确认检测到有效语音"); } if (pickPhase.value === "recording") { setPickPhase("processing", "收到ASR识别结果"); } if (streamingUserId && !completed) { currentUserSubtitle.value = subtitleContent; updateStreamingMessage("user", subtitleContent, completed); } else if (!streamingUserId) { startNewUserQuery(); currentUserSubtitle.value = subtitleContent; addChatMessage("user", subtitleContent, completed); } else { currentUserSubtitle.value = subtitleContent; updateStreamingMessage("user", subtitleContent, completed); } if (completed) { console.info("完整询问内容:", subtitleContent); const recordDuration = recordStartTime > 0 ? (Date.now() - recordStartTime) / 1000 : MIN_VALID_DURATION_SEC; recordStartTime = 0; if (!isValidVoiceInput(subtitleContent, recordDuration)) { handleInvalidVoiceInput(); } else { setPickPhase("waiting", "有效语音输入完成,等待AI回复"); options?.onUserQueryComplete?.(subtitleContent); } } } if (type === "REPLY") { if (streamingAIId && !completed) { currentAISubtitle.value = subtitleContent; updateStreamingMessage("ai", subtitleContent, completed); } else if (!streamingAIId) { startNewAIReply(); currentAISubtitle.value = subtitleContent; addChatMessage("ai", subtitleContent, completed); } else { currentAISubtitle.value = subtitleContent; updateStreamingMessage("ai", subtitleContent, completed); } // REPLY字幕完成 ≠ 播报完成,不在此处恢复拾音 // 拾音恢复由 TEXT_RENDER 的 FINISHED 回调触发 if (completed) { console.info("[字幕] AI回复字幕完成,等待播报音频结束"); } } break; } case "AUDIO_QUERY_INTERRUPT": console.info("停止发送音频"); if ( pickPhase.value === "recording" || pickPhase.value === "processing" ) { setPickPhase("idle", "音频被中断,恢复空闲"); } break; case "EMPTY_ASR_RESULT": console.info("空ASR结果,判定为无效语音输入"); if (pickPhase.value === "broadcasting") { console.info("[状态机] 播报中收到空ASR结果,忽略"); break; } // 仅在 recording/processing 状态下才处理为无效输入 // idle 状态下收到空ASR结果说明环境噪音触发,不应改变状态 if ( pickPhase.value !== "recording" && pickPhase.value !== "processing" ) { console.info(`[状态机] 当前状态 ${pickPhase.value},忽略空ASR结果`); break; } handleInvalidVoiceInput(); break; case "ASR_ERROR": console.info("ASR错误:", content); if (pickPhase.value === "broadcasting") { console.info("[状态机] 播报中收到ASR错误,忽略"); break; } // 仅在 recording/processing 状态下才处理为无效输入 if ( pickPhase.value !== "recording" && pickPhase.value !== "processing" ) { console.info(`[状态机] 当前状态 ${pickPhase.value},忽略ASR错误`); break; } handleInvalidVoiceInput(); break; default: break; } } if (status === "DH_LIB_FULL_STATUS") { const { type, action } = content; if (type === "rtcState") { if (action === "remotevideoon") { realTimeVideoReady.value = true; checkPlayUnMuteFun(); } if (action === "localVideoMuted" && content.body) { videoIsMuted.value = true; } } if (type === "wsState") { if (content.readyState === ReadyState.OPEN) { wsConnected.value = true; isConnected.value = true; } else if ( content.readyState === ReadyState.CLOSED || content.readyState === ReadyState.CLOSING ) { wsConnected.value = false; isConnected.value = false; } } } }; const checkPlayUnMuteFun = async () => { const result = await checkPlayUnMute(); videoIsMuted.value = !result; checkOver.value = true; }; const destroyCurrentInstance = async () => { if (!humanInstanceRef.value) return; try { await humanInstanceRef.value.destroy(); } catch (error) { console.error("销毁旧实例失败:", error); } finally { humanInstanceRef.value = null; isConnected.value = false; } }; const connect = async (mode: "btn" | "rtc" | "" = "btn") => { await destroyCurrentInstance(); openingPlayed.value = false; audioMode.value = mode; pullAudioFromRTC.value = mode === "rtc"; pickPhase.value = "idle"; isBroadcasting.value = false; // 获取最新 Token 并启动自动刷新 token.value = tokenManager.getToken(); tokenManager.startAutoRefresh(); const config: any = { token: token.value, wrapperId: "human-wrapper", connectParams: { ttsPer: "20007751", figureId: "4000821", resolutionHeight: 1920, resolutionWidth: 1080, inactiveDisconnectSec: 300, }, renderParams: { closeLog: true, autoChromaKey: false, fullStatus: true, }, onDigitalHumanCallback: handleDigitalHumanCallback, }; if (mode !== "") { config.connectParams.pullAudioFromRtc = mode === "rtc"; config.connectParams.pickAudioMode = mode === "btn" ? "pressButton" : "free"; config.connectParams.enableInterrupt = enableInterruptRef.value; } humanInstanceRef.value = new RealTimeHuman(config); humanInstanceRef.value.createServer(); }; const disconnect = async () => { silenceDetector.stop(); clearBroadcastTimeout(); tokenManager.stopAutoRefresh(); pickPhase.value = "idle"; isBroadcasting.value = false; currentAISubtitle.value = ""; currentUserSubtitle.value = ""; chatMessages.value = []; streamingAIId = null; streamingUserId = null; isRecording.value = false; canTransfer.value = false; isMuted.value = true; realTimeVideoReady.value = false; wsConnected.value = false; checkOver.value = false; videoIsMuted.value = false; openingPlayed.value = false; try { await humanInstanceRef.value?.destroy?.(); } catch (error) { console.error("销毁发生错误:", error); } finally { humanInstanceRef.value = null; isConnected.value = false; } }; const textRender = (text: string) => { const commandId = uuidv4(); enterBroadcasting(); humanInstanceRef.value?.sendMessage?.( { action: "TEXT_RENDER", body: text, requestId: commandId, }, ({ action }: { action: string }) => { console.info(`[播报] TEXT_RENDER callback: ${action}`); if (action === "FINISHED") { onBroadcastFinished(); } }, ); }; const streamTextRender = (texts: string[], requestId?: string) => { const commandId = requestId || uuidv4(); enterBroadcasting(); texts.forEach((text, index) => { humanInstanceRef.value?.textStreamRender?.({ body: JSON.stringify({ first: index === 0, last: index === texts.length - 1, text, }), requestId: commandId, }); }); }; const interrupt = async () => { console.info("发送打断"); clearBroadcastTimeout(); await humanInstanceRef.value?.interrupt?.(); // 记录打断时间戳,用于忽略打断后延迟到达的ASR结果 lastInterruptTime = Date.now(); // 打断后播报结束,恢复状态 if (isBroadcasting.value) { isBroadcasting.value = false; console.info("[播报] 被打断,播报终止"); } if (pickPhase.value === "broadcasting") { setPickPhase("idle", "播报被打断,恢复空闲"); if (audioMode.value === "rtc") { resumeAutoPickup(); } } console.info("打断生效"); }; const muteHuman = () => humanInstanceRef.value?.muteHuman?.(); const unMuteHuman = () => humanInstanceRef.value?.unMuteHuman?.(); const playHuman = () => humanInstanceRef.value?.playHuman?.(); const pauseHuman = () => humanInstanceRef.value?.pauseHuman?.(); const changeMicrophoneState = (mute: boolean) => { humanInstanceRef.value?.muteMicrophone?.(mute); canTransfer.value = !mute; isMuted.value = mute; if (audioMode.value === "rtc") { if (mute) { silenceDetector.pause(); } else { const resumed = silenceDetector.resume(); if (!resumed) { startSilenceDetection(); } } } }; /** * 启动静音检测(仅RTC模式) */ const startSilenceDetection = async () => { if (audioMode.value !== "rtc") return; const started = await silenceDetector.start( () => { // 状态守卫:仅在 recording 状态下才触发识别流程 // idle 状态下静音超时不应自动进入识别(用户未主动开始录入) if (pickPhase.value !== "recording") { console.info( `[自动拾音] 当前状态为 ${pickPhase.value},忽略静音超时回调`, ); return; } console.info("[自动拾音] 检测到5秒静音,暂停拾音等待ASR结果"); setPickPhase("processing", "RTC静音超时,等待ASR结果"); humanInstanceRef.value?.muteMicrophone?.(true); isMuted.value = true; canTransfer.value = false; }, () => { // 声音恢复回调:仅用于日志记录,不自动改变状态 // 状态转换由 ASR 的 QUERY 结果驱动(有实质内容才转换) // 避免环境噪音触发 idle → recording 导致误进入识别状态 console.info("[自动拾音] 检测到声音"); }, ); if (!started) { console.warn("[自动拾音] 静音检测器启动失败,可能没有麦克风权限"); } }; /** * 自动恢复拾音(播报完成后或无效输入时调用) */ const resumeAutoPickup = () => { humanInstanceRef.value?.muteMicrophone?.(false); isMuted.value = false; canTransfer.value = true; const resumed = silenceDetector.resume(); if (!resumed) { console.warn("[自动拾音] resume 失败,回退到 start 重新获取麦克风"); startSilenceDetection(); } }; const startRecord = async () => { if (isRecording.value) return; // 播报中不允许录音,防止中断数字人播报 if (isBroadcasting.value) { console.info("[状态机] 数字人播报中,不可开始录音"); return; } // waiting/broadcasting状态下不允许录音 if (pickPhase.value === "waiting" || pickPhase.value === "broadcasting") { console.info(`[状态机] 当前状态 ${pickPhase.value},不可开始录音`); return; } if (enableInterruptRef.value) { await humanInstanceRef.value?.interrupt(); } isRecording.value = true; recordStartTime = Date.now(); setPickPhase("recording", "BTN模式开始录音"); humanInstanceRef.value?.startRecord?.(); }; const stopRecord = () => { if (!isRecording.value) return; isRecording.value = false; if (pickPhase.value === "recording") { setPickPhase("processing", "BTN模式松手,等待ASR结果"); } humanInstanceRef.value?.stopRecord?.(); }; // 开场白自动播放 watch( [realTimeVideoReady, wsConnected, checkOver, videoIsMuted, audioMode], ([isVideoReady, isWsConnected, isCheckOver, isVideoMuted, mode]) => { if (mode === "rtc") { changeMicrophoneState(true); } if (openingPlayed.value) return; if (isVideoReady && isWsConnected && isCheckOver && !isVideoMuted) { openingPlayed.value = true; const commandId = uuidv4(); if (mode === "rtc") { humanInstanceRef.value?.muteMicrophone?.(true); } setTimeout(() => { humanInstanceRef.value?.sendMessage?.( { action: "TEXT_RENDER", body: DEFAULT_TEXT, requestId: commandId, }, ({ action }: { action: string }) => { if (action === "FINISHED") { isBroadcasting.value = false; if (mode === "rtc") { humanInstanceRef.value?.muteMicrophone?.(false); isMuted.value = false; setPickPhase("idle", "开场白播放完成,RTC模式就绪"); startSilenceDetection(); } else { setPickPhase("idle", "开场白播放完成"); } } }, ); }, OPENING_TTS_DELAY_MS); } }, ); // 键盘事件 const onKeyDown = (e: KeyboardEvent) => { e.preventDefault(); if (e.repeat || audioMode.value !== "btn") return; if (e.code === "Space" || e.keyCode === 32) { startRecord(); } }; const onKeyUp = (e: KeyboardEvent) => { if (e.repeat) return; if (audioMode.value === "btn" && (e.code === "Space" || e.keyCode === 32)) { stopRecord(); } }; watch( audioMode, (mode) => { if (mode === "btn") { document.addEventListener("keydown", onKeyDown); document.addEventListener("keyup", onKeyUp); return; } document.removeEventListener("keydown", onKeyDown); document.removeEventListener("keyup", onKeyUp); }, { immediate: true }, ); onBeforeUnmount(() => { document.removeEventListener("keydown", onKeyDown); document.removeEventListener("keyup", onKeyUp); silenceDetector.stop(); clearBroadcastTimeout(); tokenManager.stopAutoRefresh(); const inst = humanInstanceRef.value; humanInstanceRef.value = null; if (inst) { void inst.destroy().catch((error: unknown) => { console.error("组件卸载销毁数字人实例失败:", error); }); } }); return { // 状态 humanInstanceRef, isConnected, realTimeVideoReady, wsConnected, videoIsMuted, audioMode, isRecording, canTransfer, isMuted, token, tokenManager, // 拾音状态机 pickPhase, setPickPhase, // 播报状态 isBroadcasting, // 字幕 subtitleMode, currentAISubtitle, currentUserSubtitle, chatMessages, // 方法 connect, disconnect, textRender, streamTextRender, interrupt, muteHuman, unMuteHuman, playHuman, pauseHuman, changeMicrophoneState, startRecord, stopRecord, }; }