beba6b81ce
All checks were successful
Gitea Actions Demo / Explore-Gitea-Actions (push) Successful in 1m48s
868 lines
27 KiB
TypeScript
868 lines
27 KiB
TypeScript
import { ref, onBeforeUnmount, watch } from "vue";
|
||
import { v4 as uuidv4 } from "uuid";
|
||
import RealTimeHuman from "@bddh/starling-realtime-client";
|
||
import { checkPlayUnMute } from "@/utils/audioUtils";
|
||
import { SilenceDetector } from "@/utils/SilenceDetector";
|
||
import { TokenManager } from "@/utils/tokenManager";
|
||
import type { ChatMessage, SubtitleMode } from "@/types";
|
||
|
||
const DEFAULT_TEXT =
|
||
"你好！我是皮特教授，欢迎向我提问任何有关电池的问题。";
|
||
const OPENING_TTS_DELAY_MS = 200;
|
||
const SILENCE_TIMEOUT_MS = 5000;
|
||
// 最短有效语音时长（秒），低于此时长视为无效输入
|
||
const MIN_VALID_DURATION_SEC = 0.5;
|
||
// 播报超时保护（毫秒）：如果FINISHED回调未触发，自动恢复状态
|
||
const BROADCAST_TIMEOUT_MS = 60000;
|
||
const APPID = "i-sfyrkdr0ubepa";
|
||
const APP_KEY = "72vpdansbpg8nxjbxzhp";
|
||
// 打断词列表：识别到这些词时自动打断播报
|
||
const INTERRUPT_KEYWORDS = ["稍等一下", "停一下", "抱歉打断一下", "不好意思", "对不起", "停", "闭嘴"];
|
||
|
||
/**
|
||
* 检测文本是否包含实质内容（去除空白和标点后仍有字符）
|
||
* 用于区分真实语音输入和环境噪音触发的空白ASR结果
|
||
*/
|
||
const hasSubstantialContent = (text: string): boolean => {
|
||
const cleaned = text.replace(
|
||
/[\s\u3000.,!?;:，。！？；：、…—\-_·""''（）()\[\]【】《》<>]/g,
|
||
"",
|
||
);
|
||
return cleaned.length > 0;
|
||
};
|
||
|
||
/**
|
||
* 检测文本是否包含打断词
|
||
*/
|
||
const containsInterruptKeyword = (text: string): boolean => {
|
||
const cleaned = text.replace(
|
||
/[\s\u3000.,!?;:，。！？；：、…—\-_·""''（）()\[\]【】《》<>]/g,
|
||
"",
|
||
);
|
||
return INTERRUPT_KEYWORDS.some((keyword) => cleaned.includes(keyword));
|
||
};
|
||
|
||
enum ReadyState {
|
||
UNINSTANTIATED = -1,
|
||
CONNECTING = 0,
|
||
OPEN = 1,
|
||
CLOSING = 2,
|
||
CLOSED = 3,
|
||
}
|
||
|
||
/**
|
||
* 拾音阶段状态机
|
||
* - idle: 空闲，可开始拾音
|
||
* - recording: 正在录音（BTN模式按住/RTC模式ASR确认有效语音）
|
||
* - processing: 语音采集完成，ASR正在识别
|
||
* - waiting: 等待AI回复
|
||
* - broadcasting: 数字人正在播报，不可拾音
|
||
*
|
||
* 合法状态转换：
|
||
* idle → recording (BTN模式按下 / RTC模式ASR确认有效语音内容)
|
||
* recording → processing (BTN松手 / RTC静音超时 / 收到ASR结果)
|
||
* recording → idle (音频被中断)
|
||
* processing → waiting (有效语音输入完成)
|
||
* processing → idle (无效语音输入 / ASR错误 / 音频被中断)
|
||
* waiting → broadcasting (AI回复到达，开始播报)
|
||
* broadcasting → idle (播报完成 / 播报被打断 / 播报超时)
|
||
*
|
||
* 关键防护：
|
||
* - idle状态下仅当ASR返回包含实质内容的QUERY时才转换到recording
|
||
* - 环境噪音触发的空白/标点ASR结果在idle状态下被忽略
|
||
* - SilenceDetector声音回调不触发状态转换，仅用于静音超时检测
|
||
*/
|
||
export type PickPhase =
|
||
| "idle"
|
||
| "recording"
|
||
| "processing"
|
||
| "waiting"
|
||
| "broadcasting";
|
||
|
||
/** 合法状态转换映射表 */
|
||
const VALID_TRANSITIONS: Record<PickPhase, PickPhase[]> = {
|
||
idle: ["recording"],
|
||
recording: ["processing", "idle"],
|
||
processing: ["waiting", "idle"],
|
||
waiting: ["broadcasting", "idle"],
|
||
broadcasting: ["idle"],
|
||
};
|
||
|
||
export function useDigitalHuman(options?: {
|
||
onUserQueryComplete?: (text: string) => void;
|
||
}) {
|
||
const humanInstanceRef = ref<any>(null);
|
||
const realTimeVideoReady = ref(false);
|
||
const wsConnected = ref(false);
|
||
const videoIsMuted = ref(false);
|
||
const checkOver = ref(false);
|
||
const openingPlayed = ref(false);
|
||
const audioMode = ref<"btn" | "rtc" | "">("");
|
||
const isRecording = ref(false);
|
||
const canTransfer = ref(false);
|
||
const isMuted = ref(true);
|
||
const enableInterruptRef = ref(true);
|
||
const pullAudioFromRTC = ref(false);
|
||
const isConnected = ref(false);
|
||
|
||
// ===== 拾音状态机 =====
|
||
const pickPhase = ref<PickPhase>("idle");
|
||
|
||
const setPickPhase = (phase: PickPhase, reason: string) => {
|
||
const prev = pickPhase.value;
|
||
// 状态转换守卫：检查是否为合法转换
|
||
const allowed = VALID_TRANSITIONS[prev];
|
||
if (allowed && !allowed.includes(phase)) {
|
||
console.warn(
|
||
`[状态机] 非法状态转换: ${prev} → ${phase} (${reason})，已阻止`,
|
||
);
|
||
return;
|
||
}
|
||
pickPhase.value = phase;
|
||
console.info(`[状态机] pickPhase: ${prev} → ${phase} (${reason})`);
|
||
};
|
||
|
||
// ===== 播报状态追踪 =====
|
||
// 数字人是否正在播报（TTS语音输出中）
|
||
const isBroadcasting = ref(false);
|
||
|
||
// 字幕相关
|
||
const subtitleMode = ref<SubtitleMode>("ai-only");
|
||
const currentAISubtitle = ref("");
|
||
const currentUserSubtitle = ref("");
|
||
const chatMessages = ref<ChatMessage[]>([]);
|
||
let streamingAIId: string | null = null;
|
||
let streamingUserId: string | null = null;
|
||
|
||
// Token 配置：使用 TokenManager 自动生成与刷新
|
||
const tokenManager = new TokenManager(APPID, APP_KEY);
|
||
const token = ref(tokenManager.getToken());
|
||
|
||
// Token 更新时同步到 ref
|
||
tokenManager.onTokenRefreshed((tokenInfo) => {
|
||
token.value = tokenInfo.token;
|
||
});
|
||
|
||
// 静音检测器（RTC模式专用）
|
||
const silenceDetector = new SilenceDetector({
|
||
volumeThreshold: 8,
|
||
silenceDurationMs: SILENCE_TIMEOUT_MS,
|
||
});
|
||
|
||
// 录音时长追踪（BTN模式）
|
||
let recordStartTime = 0;
|
||
|
||
// 播报超时保护计时器
|
||
let broadcastTimeoutId: ReturnType<typeof setTimeout> | null = null;
|
||
// 打断词检测时间戳：用于忽略打断后延迟到达的ASR结果
|
||
let lastInterruptTime = 0;
|
||
|
||
const clearBroadcastTimeout = () => {
|
||
if (broadcastTimeoutId !== null) {
|
||
clearTimeout(broadcastTimeoutId);
|
||
broadcastTimeoutId = null;
|
||
}
|
||
};
|
||
|
||
const startBroadcastTimeout = () => {
|
||
clearBroadcastTimeout();
|
||
broadcastTimeoutId = setTimeout(() => {
|
||
console.warn("[播报] 超时保护触发：FINISHED回调未收到，自动恢复状态");
|
||
isBroadcasting.value = false;
|
||
if (pickPhase.value === "broadcasting") {
|
||
setPickPhase("idle", "播报超时保护，恢复空闲");
|
||
if (audioMode.value === "rtc") {
|
||
resumeAutoPickup();
|
||
}
|
||
}
|
||
}, BROADCAST_TIMEOUT_MS);
|
||
};
|
||
|
||
/**
|
||
* 进入播报状态
|
||
* RTC模式下保持麦克风开启，允许ASR识别打断词
|
||
*/
|
||
const enterBroadcasting = () => {
|
||
isBroadcasting.value = true;
|
||
setPickPhase("broadcasting", "数字人开始播报");
|
||
startBroadcastTimeout();
|
||
// RTC模式：播报期间保持麦克风开启，允许ASR识别打断词
|
||
if (audioMode.value === "rtc") {
|
||
humanInstanceRef.value?.muteMicrophone?.(false);
|
||
isMuted.value = false;
|
||
canTransfer.value = true;
|
||
const resumed = silenceDetector.resume();
|
||
if (!resumed) {
|
||
startSilenceDetection();
|
||
}
|
||
console.info("[播报] RTC模式：麦克风保持开启，允许ASR识别打断词");
|
||
}
|
||
console.info("[播报] 开始播报，等待FINISHED信号");
|
||
};
|
||
|
||
/**
|
||
* 检测语音输入是否有效
|
||
*/
|
||
const isValidVoiceInput = (text: string, durationSec: number): boolean => {
|
||
const cleaned = text.replace(
|
||
/[\s\u3000.,!?;:，。！？；：、…—\-_·""''（）()\[\]【】《》<>]/g,
|
||
"",
|
||
);
|
||
if (!cleaned) {
|
||
console.info("[语音有效性] 文本为空或仅含标点/空白，判定为无效输入");
|
||
return false;
|
||
}
|
||
if (durationSec < MIN_VALID_DURATION_SEC) {
|
||
console.info(
|
||
`[语音有效性] 录音时长 ${durationSec.toFixed(2)}s 不足 ${MIN_VALID_DURATION_SEC}s，判定为无效输入`,
|
||
);
|
||
return false;
|
||
}
|
||
return true;
|
||
};
|
||
|
||
/**
|
||
* 无效语音输入处理：重置状态，恢复可拾音
|
||
*/
|
||
const handleInvalidVoiceInput = () => {
|
||
console.info("[语音有效性] 终止当前交互流程，不进入等待回复状态");
|
||
currentUserSubtitle.value = "";
|
||
// 先暂停静音检测器，防止恢复拾音时产生竞争
|
||
silenceDetector.pause();
|
||
setPickPhase("idle", "无效语音输入，恢复空闲");
|
||
if (audioMode.value === "rtc") {
|
||
resumeAutoPickup();
|
||
}
|
||
};
|
||
|
||
/**
|
||
* 播报完成后的统一处理：恢复空闲状态，RTC模式下恢复拾音
|
||
*/
|
||
const onBroadcastFinished = () => {
|
||
clearBroadcastTimeout();
|
||
isBroadcasting.value = false;
|
||
console.info("[播报] 数字人播报完成");
|
||
// 仅在 broadcasting 状态下恢复空闲，避免状态错乱
|
||
if (pickPhase.value === "broadcasting") {
|
||
setPickPhase("idle", "播报完成，恢复空闲");
|
||
if (audioMode.value === "rtc") {
|
||
resumeAutoPickup();
|
||
}
|
||
}
|
||
};
|
||
|
||
const addChatMessage = (
|
||
role: "user" | "ai",
|
||
content: string,
|
||
completed: boolean = false,
|
||
): ChatMessage => {
|
||
const msg: ChatMessage = {
|
||
id: uuidv4(),
|
||
role,
|
||
content,
|
||
timestamp: Date.now(),
|
||
completed,
|
||
};
|
||
if (!completed) {
|
||
if (role === "ai") {
|
||
streamingAIId = msg.id;
|
||
} else {
|
||
streamingUserId = msg.id;
|
||
}
|
||
}
|
||
chatMessages.value = [...chatMessages.value, msg];
|
||
return msg;
|
||
};
|
||
|
||
const updateStreamingMessage = (
|
||
role: "user" | "ai",
|
||
content: string,
|
||
completed: boolean,
|
||
) => {
|
||
const targetId = role === "ai" ? streamingAIId : streamingUserId;
|
||
if (!targetId) return;
|
||
|
||
const idx = chatMessages.value.findIndex((m) => m.id === targetId);
|
||
if (idx === -1) return;
|
||
|
||
const updated: ChatMessage = {
|
||
...chatMessages.value[idx],
|
||
content,
|
||
completed,
|
||
};
|
||
chatMessages.value = [
|
||
...chatMessages.value.slice(0, idx),
|
||
updated,
|
||
...chatMessages.value.slice(idx + 1),
|
||
];
|
||
|
||
if (completed) {
|
||
if (role === "ai") streamingAIId = null;
|
||
else streamingUserId = null;
|
||
}
|
||
};
|
||
|
||
const startNewAIReply = () => {
|
||
currentAISubtitle.value = "";
|
||
streamingAIId = null;
|
||
};
|
||
|
||
const startNewUserQuery = () => {
|
||
currentUserSubtitle.value = "";
|
||
streamingUserId = null;
|
||
};
|
||
|
||
const handleDigitalHumanCallback = async (data: any) => {
|
||
const { status, content } = data;
|
||
console.info("[DH Callback]", status, content);
|
||
|
||
if (status === "DH_LIB_MESSAGE") {
|
||
switch (content.action) {
|
||
case "DOWN_SUBTITLE": {
|
||
const {
|
||
content: subtitleContent,
|
||
completed,
|
||
type,
|
||
} = JSON.parse(content.body);
|
||
if (type === "QUERY") {
|
||
// 播报期间检测打断词：识别到打断词后仅中断播报并恢复拾音，不调用LLM接口
|
||
if (pickPhase.value === "broadcasting") {
|
||
const hasInterrupt = containsInterruptKeyword(subtitleContent);
|
||
if (hasInterrupt) {
|
||
console.info(`[打断词检测] 播报中识别到打断词: "${subtitleContent}"，自动打断（仅中断播报，不调用LLM）`);
|
||
interrupt();
|
||
} else {
|
||
console.info(`[打断词检测] 播报中收到ASR结果: "${subtitleContent}"，非打断词，忽略`);
|
||
}
|
||
break;
|
||
}
|
||
|
||
// waiting 状态下忽略ASR结果（正在等LLM回复，不应接受新输入）
|
||
if (pickPhase.value === "waiting") {
|
||
console.info("[状态机] 等待AI回复中，忽略QUERY字幕");
|
||
break;
|
||
}
|
||
|
||
// idle 状态下收到 QUERY：必须验证内容有实质才允许状态转换
|
||
// 环境噪音可能触发 ASR 产生空白/标点内容的 QUERY，不应进入识别
|
||
if (pickPhase.value === "idle") {
|
||
// 忽略打断后3秒内到达的延迟ASR结果（避免打断词被当作新输入）
|
||
if (Date.now() - lastInterruptTime < 3000) {
|
||
console.info(
|
||
"[状态机] 打断后延迟到达的ASR结果，忽略",
|
||
);
|
||
break;
|
||
}
|
||
const hasSubstance = hasSubstantialContent(subtitleContent);
|
||
if (!hasSubstance) {
|
||
console.info(
|
||
"[状态机] idle状态下QUERY内容无实质（空白/标点），忽略",
|
||
);
|
||
break;
|
||
}
|
||
// 内容有实质：用户已开始有效说话
|
||
// 暂停静音检测器避免竞争，然后进入 recording 再到 processing
|
||
if (audioMode.value === "rtc") {
|
||
silenceDetector.pause();
|
||
}
|
||
console.info(
|
||
"[状态机] idle状态下收到有效ASR结果，用户已开始说话",
|
||
);
|
||
setPickPhase("recording", "ASR确认检测到有效语音");
|
||
}
|
||
|
||
if (pickPhase.value === "recording") {
|
||
setPickPhase("processing", "收到ASR识别结果");
|
||
}
|
||
|
||
if (streamingUserId && !completed) {
|
||
currentUserSubtitle.value = subtitleContent;
|
||
updateStreamingMessage("user", subtitleContent, completed);
|
||
} else if (!streamingUserId) {
|
||
startNewUserQuery();
|
||
currentUserSubtitle.value = subtitleContent;
|
||
addChatMessage("user", subtitleContent, completed);
|
||
} else {
|
||
currentUserSubtitle.value = subtitleContent;
|
||
updateStreamingMessage("user", subtitleContent, completed);
|
||
}
|
||
if (completed) {
|
||
console.info("完整询问内容：", subtitleContent);
|
||
const recordDuration =
|
||
recordStartTime > 0
|
||
? (Date.now() - recordStartTime) / 1000
|
||
: MIN_VALID_DURATION_SEC;
|
||
recordStartTime = 0;
|
||
if (!isValidVoiceInput(subtitleContent, recordDuration)) {
|
||
handleInvalidVoiceInput();
|
||
} else {
|
||
setPickPhase("waiting", "有效语音输入完成，等待AI回复");
|
||
options?.onUserQueryComplete?.(subtitleContent);
|
||
}
|
||
}
|
||
}
|
||
if (type === "REPLY") {
|
||
if (streamingAIId && !completed) {
|
||
currentAISubtitle.value = subtitleContent;
|
||
updateStreamingMessage("ai", subtitleContent, completed);
|
||
} else if (!streamingAIId) {
|
||
startNewAIReply();
|
||
currentAISubtitle.value = subtitleContent;
|
||
addChatMessage("ai", subtitleContent, completed);
|
||
} else {
|
||
currentAISubtitle.value = subtitleContent;
|
||
updateStreamingMessage("ai", subtitleContent, completed);
|
||
}
|
||
// REPLY字幕完成 ≠ 播报完成，不在此处恢复拾音
|
||
// 拾音恢复由 TEXT_RENDER 的 FINISHED 回调触发
|
||
if (completed) {
|
||
console.info("[字幕] AI回复字幕完成，等待播报音频结束");
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "AUDIO_QUERY_INTERRUPT":
|
||
console.info("停止发送音频");
|
||
if (
|
||
pickPhase.value === "recording" ||
|
||
pickPhase.value === "processing"
|
||
) {
|
||
setPickPhase("idle", "音频被中断，恢复空闲");
|
||
}
|
||
break;
|
||
case "EMPTY_ASR_RESULT":
|
||
console.info("空ASR结果，判定为无效语音输入");
|
||
if (pickPhase.value === "broadcasting") {
|
||
console.info("[状态机] 播报中收到空ASR结果，忽略");
|
||
break;
|
||
}
|
||
// 仅在 recording/processing 状态下才处理为无效输入
|
||
// idle 状态下收到空ASR结果说明环境噪音触发，不应改变状态
|
||
if (
|
||
pickPhase.value !== "recording" &&
|
||
pickPhase.value !== "processing"
|
||
) {
|
||
console.info(`[状态机] 当前状态 ${pickPhase.value}，忽略空ASR结果`);
|
||
break;
|
||
}
|
||
handleInvalidVoiceInput();
|
||
break;
|
||
case "ASR_ERROR":
|
||
console.info("ASR错误:", content);
|
||
if (pickPhase.value === "broadcasting") {
|
||
console.info("[状态机] 播报中收到ASR错误，忽略");
|
||
break;
|
||
}
|
||
// 仅在 recording/processing 状态下才处理为无效输入
|
||
if (
|
||
pickPhase.value !== "recording" &&
|
||
pickPhase.value !== "processing"
|
||
) {
|
||
console.info(`[状态机] 当前状态 ${pickPhase.value}，忽略ASR错误`);
|
||
break;
|
||
}
|
||
handleInvalidVoiceInput();
|
||
break;
|
||
default:
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (status === "DH_LIB_FULL_STATUS") {
|
||
const { type, action } = content;
|
||
if (type === "rtcState") {
|
||
if (action === "remotevideoon") {
|
||
realTimeVideoReady.value = true;
|
||
checkPlayUnMuteFun();
|
||
}
|
||
if (action === "localVideoMuted" && content.body) {
|
||
videoIsMuted.value = true;
|
||
}
|
||
}
|
||
if (type === "wsState") {
|
||
if (content.readyState === ReadyState.OPEN) {
|
||
wsConnected.value = true;
|
||
isConnected.value = true;
|
||
} else if (
|
||
content.readyState === ReadyState.CLOSED ||
|
||
content.readyState === ReadyState.CLOSING
|
||
) {
|
||
wsConnected.value = false;
|
||
isConnected.value = false;
|
||
}
|
||
}
|
||
}
|
||
};
|
||
|
||
const checkPlayUnMuteFun = async () => {
|
||
const result = await checkPlayUnMute();
|
||
videoIsMuted.value = !result;
|
||
checkOver.value = true;
|
||
};
|
||
|
||
const destroyCurrentInstance = async () => {
|
||
if (!humanInstanceRef.value) return;
|
||
try {
|
||
await humanInstanceRef.value.destroy();
|
||
} catch (error) {
|
||
console.error("销毁旧实例失败:", error);
|
||
} finally {
|
||
humanInstanceRef.value = null;
|
||
isConnected.value = false;
|
||
}
|
||
};
|
||
|
||
const connect = async (mode: "btn" | "rtc" | "" = "btn") => {
|
||
await destroyCurrentInstance();
|
||
openingPlayed.value = false;
|
||
audioMode.value = mode;
|
||
pullAudioFromRTC.value = mode === "rtc";
|
||
pickPhase.value = "idle";
|
||
isBroadcasting.value = false;
|
||
|
||
// 获取最新 Token 并启动自动刷新
|
||
token.value = tokenManager.getToken();
|
||
tokenManager.startAutoRefresh();
|
||
|
||
const config: any = {
|
||
token: token.value,
|
||
wrapperId: "human-wrapper",
|
||
connectParams: {
|
||
ttsPer: "20007751",
|
||
figureId: "4000821",
|
||
resolutionHeight: 1920,
|
||
resolutionWidth: 1080,
|
||
inactiveDisconnectSec: 300,
|
||
},
|
||
renderParams: {
|
||
closeLog: true,
|
||
autoChromaKey: false,
|
||
fullStatus: true,
|
||
},
|
||
onDigitalHumanCallback: handleDigitalHumanCallback,
|
||
};
|
||
|
||
if (mode !== "") {
|
||
config.connectParams.pullAudioFromRtc = mode === "rtc";
|
||
config.connectParams.pickAudioMode =
|
||
mode === "btn" ? "pressButton" : "free";
|
||
config.connectParams.enableInterrupt = enableInterruptRef.value;
|
||
}
|
||
|
||
humanInstanceRef.value = new RealTimeHuman(config);
|
||
humanInstanceRef.value.createServer();
|
||
};
|
||
|
||
const disconnect = async () => {
|
||
silenceDetector.stop();
|
||
clearBroadcastTimeout();
|
||
tokenManager.stopAutoRefresh();
|
||
pickPhase.value = "idle";
|
||
isBroadcasting.value = false;
|
||
|
||
currentAISubtitle.value = "";
|
||
currentUserSubtitle.value = "";
|
||
chatMessages.value = [];
|
||
streamingAIId = null;
|
||
streamingUserId = null;
|
||
isRecording.value = false;
|
||
canTransfer.value = false;
|
||
isMuted.value = true;
|
||
realTimeVideoReady.value = false;
|
||
wsConnected.value = false;
|
||
checkOver.value = false;
|
||
videoIsMuted.value = false;
|
||
openingPlayed.value = false;
|
||
try {
|
||
await humanInstanceRef.value?.destroy?.();
|
||
} catch (error) {
|
||
console.error("销毁发生错误：", error);
|
||
} finally {
|
||
humanInstanceRef.value = null;
|
||
isConnected.value = false;
|
||
}
|
||
};
|
||
|
||
const textRender = (text: string) => {
|
||
const commandId = uuidv4();
|
||
enterBroadcasting();
|
||
|
||
humanInstanceRef.value?.sendMessage?.(
|
||
{
|
||
action: "TEXT_RENDER",
|
||
body: text,
|
||
requestId: commandId,
|
||
},
|
||
({ action }: { action: string }) => {
|
||
console.info(`[播报] TEXT_RENDER callback: ${action}`);
|
||
if (action === "FINISHED") {
|
||
onBroadcastFinished();
|
||
}
|
||
},
|
||
);
|
||
};
|
||
|
||
const streamTextRender = (texts: string[], requestId?: string) => {
|
||
const commandId = requestId || uuidv4();
|
||
enterBroadcasting();
|
||
|
||
texts.forEach((text, index) => {
|
||
humanInstanceRef.value?.textStreamRender?.({
|
||
body: JSON.stringify({
|
||
first: index === 0,
|
||
last: index === texts.length - 1,
|
||
text,
|
||
}),
|
||
requestId: commandId,
|
||
});
|
||
});
|
||
};
|
||
|
||
const interrupt = async () => {
|
||
console.info("发送打断");
|
||
clearBroadcastTimeout();
|
||
await humanInstanceRef.value?.interrupt?.();
|
||
// 记录打断时间戳，用于忽略打断后延迟到达的ASR结果
|
||
lastInterruptTime = Date.now();
|
||
// 打断后播报结束，恢复状态
|
||
if (isBroadcasting.value) {
|
||
isBroadcasting.value = false;
|
||
console.info("[播报] 被打断，播报终止");
|
||
}
|
||
if (pickPhase.value === "broadcasting") {
|
||
setPickPhase("idle", "播报被打断，恢复空闲");
|
||
if (audioMode.value === "rtc") {
|
||
resumeAutoPickup();
|
||
}
|
||
}
|
||
console.info("打断生效");
|
||
};
|
||
|
||
const muteHuman = () => humanInstanceRef.value?.muteHuman?.();
|
||
const unMuteHuman = () => humanInstanceRef.value?.unMuteHuman?.();
|
||
const playHuman = () => humanInstanceRef.value?.playHuman?.();
|
||
const pauseHuman = () => humanInstanceRef.value?.pauseHuman?.();
|
||
|
||
const changeMicrophoneState = (mute: boolean) => {
|
||
humanInstanceRef.value?.muteMicrophone?.(mute);
|
||
canTransfer.value = !mute;
|
||
isMuted.value = mute;
|
||
|
||
if (audioMode.value === "rtc") {
|
||
if (mute) {
|
||
silenceDetector.pause();
|
||
} else {
|
||
const resumed = silenceDetector.resume();
|
||
if (!resumed) {
|
||
startSilenceDetection();
|
||
}
|
||
}
|
||
}
|
||
};
|
||
|
||
/**
|
||
* 启动静音检测（仅RTC模式）
|
||
*/
|
||
const startSilenceDetection = async () => {
|
||
if (audioMode.value !== "rtc") return;
|
||
|
||
const started = await silenceDetector.start(
|
||
() => {
|
||
// 状态守卫：仅在 recording 状态下才触发识别流程
|
||
// idle 状态下静音超时不应自动进入识别（用户未主动开始录入）
|
||
if (pickPhase.value !== "recording") {
|
||
console.info(
|
||
`[自动拾音] 当前状态为 ${pickPhase.value}，忽略静音超时回调`,
|
||
);
|
||
return;
|
||
}
|
||
|
||
console.info("[自动拾音] 检测到5秒静音，暂停拾音等待ASR结果");
|
||
setPickPhase("processing", "RTC静音超时，等待ASR结果");
|
||
|
||
humanInstanceRef.value?.muteMicrophone?.(true);
|
||
isMuted.value = true;
|
||
canTransfer.value = false;
|
||
},
|
||
() => {
|
||
// 声音恢复回调：仅用于日志记录，不自动改变状态
|
||
// 状态转换由 ASR 的 QUERY 结果驱动（有实质内容才转换）
|
||
// 避免环境噪音触发 idle → recording 导致误进入识别状态
|
||
console.info("[自动拾音] 检测到声音");
|
||
},
|
||
);
|
||
|
||
if (!started) {
|
||
console.warn("[自动拾音] 静音检测器启动失败，可能没有麦克风权限");
|
||
}
|
||
};
|
||
|
||
/**
|
||
* 自动恢复拾音（播报完成后或无效输入时调用）
|
||
*/
|
||
const resumeAutoPickup = () => {
|
||
humanInstanceRef.value?.muteMicrophone?.(false);
|
||
isMuted.value = false;
|
||
canTransfer.value = true;
|
||
|
||
const resumed = silenceDetector.resume();
|
||
if (!resumed) {
|
||
console.warn("[自动拾音] resume 失败，回退到 start 重新获取麦克风");
|
||
startSilenceDetection();
|
||
}
|
||
};
|
||
|
||
const startRecord = async () => {
|
||
if (isRecording.value) return;
|
||
// 播报中不允许录音，防止中断数字人播报
|
||
if (isBroadcasting.value) {
|
||
console.info("[状态机] 数字人播报中，不可开始录音");
|
||
return;
|
||
}
|
||
// waiting/broadcasting状态下不允许录音
|
||
if (pickPhase.value === "waiting" || pickPhase.value === "broadcasting") {
|
||
console.info(`[状态机] 当前状态 ${pickPhase.value}，不可开始录音`);
|
||
return;
|
||
}
|
||
if (enableInterruptRef.value) {
|
||
await humanInstanceRef.value?.interrupt();
|
||
}
|
||
isRecording.value = true;
|
||
recordStartTime = Date.now();
|
||
setPickPhase("recording", "BTN模式开始录音");
|
||
humanInstanceRef.value?.startRecord?.();
|
||
};
|
||
|
||
const stopRecord = () => {
|
||
if (!isRecording.value) return;
|
||
isRecording.value = false;
|
||
if (pickPhase.value === "recording") {
|
||
setPickPhase("processing", "BTN模式松手，等待ASR结果");
|
||
}
|
||
humanInstanceRef.value?.stopRecord?.();
|
||
};
|
||
|
||
// 开场白自动播放
|
||
watch(
|
||
[realTimeVideoReady, wsConnected, checkOver, videoIsMuted, audioMode],
|
||
([isVideoReady, isWsConnected, isCheckOver, isVideoMuted, mode]) => {
|
||
if (mode === "rtc") {
|
||
changeMicrophoneState(true);
|
||
}
|
||
if (openingPlayed.value) return;
|
||
if (isVideoReady && isWsConnected && isCheckOver && !isVideoMuted) {
|
||
openingPlayed.value = true;
|
||
const commandId = uuidv4();
|
||
if (mode === "rtc") {
|
||
humanInstanceRef.value?.muteMicrophone?.(true);
|
||
}
|
||
setTimeout(() => {
|
||
humanInstanceRef.value?.sendMessage?.(
|
||
{
|
||
action: "TEXT_RENDER",
|
||
body: DEFAULT_TEXT,
|
||
requestId: commandId,
|
||
},
|
||
({ action }: { action: string }) => {
|
||
if (action === "FINISHED") {
|
||
isBroadcasting.value = false;
|
||
if (mode === "rtc") {
|
||
humanInstanceRef.value?.muteMicrophone?.(false);
|
||
isMuted.value = false;
|
||
setPickPhase("idle", "开场白播放完成，RTC模式就绪");
|
||
startSilenceDetection();
|
||
} else {
|
||
setPickPhase("idle", "开场白播放完成");
|
||
}
|
||
}
|
||
},
|
||
);
|
||
}, OPENING_TTS_DELAY_MS);
|
||
}
|
||
},
|
||
);
|
||
|
||
// 键盘事件
|
||
const onKeyDown = (e: KeyboardEvent) => {
|
||
e.preventDefault();
|
||
if (e.repeat || audioMode.value !== "btn") return;
|
||
if (e.code === "Space" || e.keyCode === 32) {
|
||
startRecord();
|
||
}
|
||
};
|
||
|
||
const onKeyUp = (e: KeyboardEvent) => {
|
||
if (e.repeat) return;
|
||
if (audioMode.value === "btn" && (e.code === "Space" || e.keyCode === 32)) {
|
||
stopRecord();
|
||
}
|
||
};
|
||
|
||
watch(
|
||
audioMode,
|
||
(mode) => {
|
||
if (mode === "btn") {
|
||
document.addEventListener("keydown", onKeyDown);
|
||
document.addEventListener("keyup", onKeyUp);
|
||
return;
|
||
}
|
||
document.removeEventListener("keydown", onKeyDown);
|
||
document.removeEventListener("keyup", onKeyUp);
|
||
},
|
||
{ immediate: true },
|
||
);
|
||
|
||
onBeforeUnmount(() => {
|
||
document.removeEventListener("keydown", onKeyDown);
|
||
document.removeEventListener("keyup", onKeyUp);
|
||
silenceDetector.stop();
|
||
clearBroadcastTimeout();
|
||
tokenManager.stopAutoRefresh();
|
||
const inst = humanInstanceRef.value;
|
||
humanInstanceRef.value = null;
|
||
if (inst) {
|
||
void inst.destroy().catch((error: unknown) => {
|
||
console.error("组件卸载销毁数字人实例失败:", error);
|
||
});
|
||
}
|
||
});
|
||
|
||
return {
|
||
// 状态
|
||
humanInstanceRef,
|
||
isConnected,
|
||
realTimeVideoReady,
|
||
wsConnected,
|
||
videoIsMuted,
|
||
audioMode,
|
||
isRecording,
|
||
canTransfer,
|
||
isMuted,
|
||
token,
|
||
tokenManager,
|
||
// 拾音状态机
|
||
pickPhase,
|
||
setPickPhase,
|
||
// 播报状态
|
||
isBroadcasting,
|
||
// 字幕
|
||
subtitleMode,
|
||
currentAISubtitle,
|
||
currentUserSubtitle,
|
||
chatMessages,
|
||
// 方法
|
||
connect,
|
||
disconnect,
|
||
textRender,
|
||
streamTextRender,
|
||
interrupt,
|
||
muteHuman,
|
||
unMuteHuman,
|
||
playHuman,
|
||
pauseHuman,
|
||
changeMicrophoneState,
|
||
startRecord,
|
||
stopRecord,
|
||
};
|
||
}
|