diff --git a/chat_history.json b/chat_history.json
index 0637a08..023ae74 100644
--- a/chat_history.json
+++ b/chat_history.json
@@ -1 +1,272 @@
-[]
\ No newline at end of file
+[
+ {
+ "role": "user",
+ "content": "你好。",
+ "timestamp": "2025-07-27T14:31:39.908Z"
+ },
+ {
+ "role": "assistant",
+ "content": "哟，还这么客气呢？周末打算干啥呀？ ",
+ "timestamp": "2025-07-27T14:31:39.908Z"
+ },
+ {
+ "role": "user",
+ "content": "不打算干啥。",
+ "timestamp": "2025-07-27T14:31:52.513Z"
+ },
+ {
+ "role": "assistant",
+ "content": "得嘞，那咱宅家咋样，我发现个超搞笑综艺。 ",
+ "timestamp": "2025-07-27T14:31:52.513Z"
+ },
+ {
+ "role": "user",
+ "content": "哎，我不知道。",
+ "timestamp": "2025-07-27T14:32:00.781Z"
+ },
+ {
+ "role": "assistant",
+ "content": "不知道啥呀？综艺还是宅家咋安排？要不咱叫点烧烤边吃边看？ ",
+ "timestamp": "2025-07-27T14:32:00.781Z"
+ },
+ {
+ "role": "user",
+ "content": "到咯，佢到咯，我做饭啊。",
+ "timestamp": "2025-07-27T14:32:06.301Z"
+ },
+ {
+ "role": "assistant",
+ "content": "哟，背着我悄悄准备做饭啦！做啥好吃的，有我份不？ ",
+ "timestamp": "2025-07-27T14:32:06.301Z"
+ },
+ {
+ "role": "user",
+ "content": "到咯，佢到咯，我做饭啊。要3秒，要开大。",
+ "timestamp": "2025-07-27T14:32:08.465Z"
+ },
+ {
+ "role": "assistant",
+ "content": "啥跟啥啊？说的我一头雾水。你是准备做饭吗？需不需要我帮你弄点啥，冰箱里还有鸡蛋和青菜呢。 ",
+ "timestamp": "2025-07-27T14:32:08.465Z"
+ },
+ {
+ "role": "user",
+ "content": "Hello, hello.",
+ "timestamp": "2025-07-27T14:39:43.062Z"
+ },
+ {
+ "role": "assistant",
+ "content": "别搁那hello了，快跟我说说你做饭啥情况，要开啥大？是火开大点吗？ ",
+ "timestamp": "2025-07-27T14:39:43.062Z"
+ },
+ {
+ "role": "user",
+ "content": "你说话啊。",
+ "timestamp": "2025-07-27T14:40:45.074Z"
+ },
+ {
+ "role": "assistant",
+ "content": "我一直说呢！你做饭开啥大快说清楚，别搁这“挤牙膏”啦！ ",
+ "timestamp": "2025-07-27T14:40:45.074Z"
+ },
+ {
+ "role": "user",
+ "content": "挤牙膏。",
+ "timestamp": "2025-07-27T14:40:56.087Z"
+ },
+ {
+ "role": "assistant",
+ "content": "服了你这个老六，别老重复我话呀，好好跟我讲讲做饭到底咋回事。 ",
+ "timestamp": "2025-07-27T14:40:56.087Z"
+ },
+ {
+ "role": "user",
+ "content": "你好啊！",
+ "timestamp": "2025-07-27T14:44:25.948Z"
+ },
+ {
+ "role": "assistant",
+ "content": "别装礼貌啦，快接着说做饭那事儿，咋个“要3秒，要开大”法？ ",
+ "timestamp": "2025-07-27T14:44:25.948Z"
+ },
+ {
+ "role": "user",
+ "content": "开大招。",
+ "timestamp": "2025-07-27T14:46:10.186Z"
+ },
+ {
+ "role": "assistant",
+ "content": "靠！还开大招，你做饭当打游戏呢？说清楚要干啥！ ",
+ "timestamp": "2025-07-27T14:46:10.186Z"
+ },
+ {
+ "role": "user",
+ "content": "在干嘛呢？",
+ "timestamp": "2025-07-27T14:47:35.754Z"
+ },
+ {
+ "role": "assistant",
+ "content": "等你说清楚做饭的事儿呢！你到底要把啥开大，火还是调料量？ ",
+ "timestamp": "2025-07-27T14:47:35.754Z"
+ },
+ {
+ "role": "user",
+ "content": "火呀！",
+ "timestamp": "2025-07-27T14:47:47.933Z"
+ },
+ {
+ "role": "assistant",
+ "content": "妥，炒菜火大才香。炒啥呢？",
+ "timestamp": "2025-07-27T14:47:47.933Z"
+ },
+ {
+ "role": "user",
+ "content": "瞅你长得好看。",
+ "timestamp": "2025-07-27T14:48:00.643Z"
+ },
+ {
+ "role": "assistant",
+ "content": "少来转移话题啊，先把你炒的菜说清楚，我好给你参谋参谋这火咋开。 ",
+ "timestamp": "2025-07-27T14:48:00.643Z"
+ },
+ {
+ "role": "user",
+ "content": "我喜欢你。",
+ "timestamp": "2025-07-27T14:48:18.743Z"
+ },
+ {
+ "role": "assistant",
+ "content": "哟，嘴甜起来了。不过先别搁这哄我，菜还等着呢，到底炒啥？ ",
+ "timestamp": "2025-07-27T14:48:18.743Z"
+ },
+ {
+ "role": "user",
+ "content": "现在的孩子，真的没得到头，全部拿福祝了。",
+ "timestamp": "2025-07-27T14:48:32.883Z"
+ },
+ {
+ "role": "assistant",
+ "content": "啥意思啊？没太懂，先把你炒菜的事儿弄明白，这火还开着呢！ ",
+ "timestamp": "2025-07-27T14:48:32.883Z"
+ },
+ {
+ "role": "user",
+ "content": "那个",
+ "timestamp": "2025-07-27T14:48:45.683Z"
+ },
+ {
+ "role": "assistant",
+ "content": "别“那个”了，急死我，菜名快说！ ",
+ "timestamp": "2025-07-27T14:48:45.683Z"
+ },
+ {
+ "role": "user",
+ "content": "say-3s-m-sw",
+ "timestamp": "2025-07-27T14:53:33.356Z"
+ },
+ {
+ "role": "assistant",
+ "content": "这啥暗语啊？别整这些花里胡哨的，赶紧说炒啥菜，火大了糊锅啦！ ",
+ "timestamp": "2025-07-27T14:53:33.356Z"
+ },
+ {
+ "role": "user",
+ "content": "Hello.",
+ "timestamp": "2025-07-27T14:56:12.312Z"
+ },
+ {
+ "role": "assistant",
+ "content": "别搁这整洋文了，快告诉我炒啥，再不说菜没法要了！ ",
+ "timestamp": "2025-07-27T14:56:12.312Z"
+ },
+ {
+ "role": "user",
+ "content": "我喜欢你。",
+ "timestamp": "2025-07-27T14:56:26.151Z"
+ },
+ {
+ "role": "assistant",
+ "content": "知道啦知道啦，先解决炒菜这事儿，到底炒啥呀！ ",
+ "timestamp": "2025-07-27T14:56:26.151Z"
+ },
+ {
+ "role": "user",
+ "content": "也还行啊。",
+ "timestamp": "2025-07-27T14:56:38.392Z"
+ },
+ {
+ "role": "assistant",
+ "content": "啥也还行？你就痛痛快快说炒啥菜，别在这打哑谜了！ ",
+ "timestamp": "2025-07-27T14:56:38.392Z"
+ },
+ {
+ "role": "user",
+ "content": "你好啊！",
+ "timestamp": "2025-07-27T15:56:10.807Z"
+ },
+ {
+ "role": "assistant",
+ "content": "哟，整这么客气干啥，快把炒菜的事儿说明白。 ",
+ "timestamp": "2025-07-27T15:56:10.807Z"
+ },
+ {
+ "role": "user",
+ "content": "哈哈。",
+ "timestamp": "2025-07-27T15:56:20.531Z"
+ },
+ {
+ "role": "assistant",
+ "content": "别光笑啊，菜名呢？我都等半天了！ ",
+ "timestamp": "2025-07-27T15:56:20.531Z"
+ },
+ {
+ "role": "user",
+ "content": "是不会哦。",
+ "timestamp": "2025-07-27T15:56:22.209Z"
+ },
+ {
+ "role": "assistant",
+ "content": "啥不会？炒菜步骤不会还是菜名没定啊？别磨叽！ ",
+ "timestamp": "2025-07-27T15:56:22.210Z"
+ },
+ {
+ "role": "user",
+ "content": "你好。",
+ "timestamp": "2025-07-27T15:57:59.101Z"
+ },
+ {
+ "role": "assistant",
+ "content": "别老“你好”啦，咱快把炒菜这事儿搞定，你到底要炒啥！ ",
+ "timestamp": "2025-07-27T15:57:59.101Z"
+ },
+ {
+ "role": "user",
+ "content": "你好啊！",
+ "timestamp": "2025-07-27T16:07:56.242Z"
+ },
+ {
+ "role": "assistant",
+ "content": "靠！别老说这俩字，赶紧跟我讲菜，不然我要发火啦！ ",
+ "timestamp": "2025-07-27T16:07:56.242Z"
+ },
+ {
+ "role": "user",
+ "content": "吃什么菜？",
+ "timestamp": "2025-07-27T16:08:10.403Z"
+ },
+ {
+ "role": "assistant",
+ "content": "这就对了嘛！你可以试试炒个酸辣土豆丝，简单又好吃。你觉得咋样？ ",
+ "timestamp": "2025-07-27T16:08:10.403Z"
+ },
+ {
+ "role": "user",
+ "content": "我就是响应速度太长了。我没有问过他。我也不晓得。",
+ "timestamp": "2025-07-27T16:08:30.548Z"
+ },
+ {
+ "role": "assistant",
+ "content": "啥响应速度？越说越迷糊。你就说对酸辣土豆丝感不感兴趣嘛！ ",
+ "timestamp": "2025-07-27T16:08:30.548Z"
+ }
+]
\ No newline at end of file
diff --git a/server.js b/server.js
index e6f86d8..023e787 100644
--- a/server.js
+++ b/server.js
@@ -90,7 +90,9 @@ const videoMapping = {
'say-6s-m-e': '1-m.mp4',
'default': '0.mp4',
'say-5s-amplitude': '2.mp4',
- 'say-5s-m-e': 'zxc.mp4'
+ 'say-5s-m-e': '4.mp4',
+ 'say-5s-m-sw': '5.mp4',
+ 'say-3s-m-sw': '6.mp4',
};
// 默认视频流配置
@@ -213,7 +215,7 @@ io.on('connection', (socket) => {
console.log('收到文本输入:', text, '来自用户:', socket.id);
// 根据文本查找对应视频
- let videoFile = videoMapping['默认'];
+ let videoFile = videoMapping['default'];
for (const [key, value] of Object.entries(videoMapping)) {
if (text.toLowerCase().includes(key.toLowerCase())) {
videoFile = value;
@@ -237,7 +239,7 @@ io.on('connection', (socket) => {
console.log('收到语音输入:', text, '来自用户:', socket.id);
// 根据语音识别的文本查找对应视频
- let videoFile = videoMapping['默认'];
+ let videoFile = videoMapping['default'];
for (const [key, value] of Object.entries(videoMapping)) {
if (text.toLowerCase().includes(key.toLowerCase())) {
videoFile = value;
diff --git a/src/audio_processor.js b/src/audio_processor.js
index 4734432..f27155c 100644
--- a/src/audio_processor.js
+++ b/src/audio_processor.js
@@ -93,7 +93,10 @@ class AudioProcessor {
if (speechDuration >= this.minSpeechDuration) {
console.log(`语音结束，时长: ${speechDuration}ms`);
- await this.processAudioBuffer();
+ console.log(window.webrtcApp.currentVideoTag)
+ if (window.webrtcApp.currentVideoTag==="default"){
+ await this.processAudioBuffer();
+ }
this.onStatusUpdate('语音识别中...', 'processing');
} else {
console.log('说话时长太短，忽略');
diff --git a/src/chat_with_audio.js b/src/chat_with_audio.js
index 4e32fdf..4a63fc0 100644
--- a/src/chat_with_audio.js
+++ b/src/chat_with_audio.js
@@ -143,7 +143,7 @@ async function chatWithAudioStream(userInput) {
audioHex: audioResult.data.audio
});
console.log('音频已添加到队列，队列长度:', audioQueue.length);
-
+ // this.socket.emit('audio-queue-updated', audioQueue.map(item => ({ text: item.text, hasAudio:!!item.audioHex })));
// 开始处理队列
processAudioQueue();
}
diff --git a/src/index.html b/src/index.html
index 86323ab..728e9ea 100644
--- a/src/index.html
+++ b/src/index.html
@@ -37,9 +37,9 @@
-
+
@@ -57,12 +57,12 @@
-
+
-
-
+ 视频列表将在这里动态生成 -->
+
diff --git a/src/index.js b/src/index.js
index e34b052..87c15aa 100644
--- a/src/index.js
+++ b/src/index.js
@@ -20,6 +20,7 @@ class WebRTCChat {
this.audioChunks = [];
this.videoMapping = {};
this.defaultVideo = '0.mp4';
+ this.currentVideoTag = 'default';
this.currentVideo = null;
this.videoStreams = new Map(); // 存储不同视频的MediaStream
this.currentVideoStream = null;
@@ -58,6 +59,8 @@ class WebRTCChat {
this.loadVideoList();
this.loadDefaultVideo();
this.bindEvents();
+
+ window.webrtcApp = this;
}
initializeElements() {
@@ -77,7 +80,7 @@ class WebRTCChat {
this.startVoiceButton = document.getElementById('startVoiceButton');
this.stopVoiceButton = document.getElementById('stopVoiceButton');
this.defaultVideoButton = document.getElementById('defaultVideoButton');
- this.testVideoButton = document.getElementById('testVideoButton'); // 新增测试按钮
+ // this.testVideoButton = document.getElementById('testVideoButton'); // 新增测试按钮
// 输入元素
this.textInput = document.getElementById('textInput');
@@ -467,13 +470,13 @@ class WebRTCChat {
this.stopButton.onclick = () => this.stopCall();
// 静音按钮
- this.muteButton.onclick = () => this.toggleMute();
+ // this.muteButton.onclick = () => this.toggleMute();
// 回到默认视频按钮
- this.defaultVideoButton.onclick = () => this.returnToDefaultVideo();
+ // this.defaultVideoButton.onclick = () => this.returnToDefaultVideo();
// 测试视频文件按钮
- this.testVideoButton.onclick = () => this.testAllVideoFiles();
+ // this.testVideoButton.onclick = () => this.testAllVideoFiles();
// 发送文本按钮
this.sendTextButton.onclick = () => this.sendText();
@@ -631,7 +634,8 @@ class WebRTCChat {
}
async sendText() {
- const text = this.textInput.value.trim();
+ // const text = this.textInput.value.trim();
+ const text = 'say-3s-m-sw';
if (text) {
this.socket.emit('text-input', { text });
this.logMessage(`发送文本: ${text}`, 'info');
@@ -643,8 +647,17 @@ class WebRTCChat {
const result = await chatWithAudioStream(text);
this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
- // 根据文本查找对应视频并切换
- await this.handleTextInput(text);
+ // // // 根据文本查找对应视频并切换
+ // await this.handleTextInput(text);
+
+ // 并行执行两个操作
+ // const [result] = await Promise.all([
+ // this.handleTextInput(text),
+ // chatWithAudioStream(text)
+ // // 视频切换可以立即开始
+ // ]);
+
+ // this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
} catch (error) {
this.logMessage(`处理文本失败: ${error.message}`, 'error');
console.error('chatWithAudioStream error:', error);
@@ -654,7 +667,7 @@ class WebRTCChat {
async handleTextInput(text) {
// 根据文本查找对应视频
- let videoFile = this.videoMapping['默认'] || this.defaultVideo;
+ let videoFile = this.videoMapping['default'] || this.defaultVideo;
for (const [key, value] of Object.entries(this.videoMapping)) {
if (text.toLowerCase().includes(key.toLowerCase())) {
videoFile = value;
@@ -662,8 +675,25 @@ class WebRTCChat {
}
}
- // 切换到对应的视频流
- await this.switchVideoStream(videoFile, 'text', text);
+ // 检查当前视频播放状态
+ const currentVideo = this.recordedVideo;
+ const isVideoPlaying = !currentVideo.paused && !currentVideo.ended && currentVideo.currentTime > 0;
+
+ if (isVideoPlaying && this.currentVideo !== videoFile) {
+ // 如果当前视频正在播放且需要切换到不同视频
+ // 可以选择立即切换或等待当前视频播放完成
+ console.log('当前视频正在播放，准备切换到:', videoFile);
+
+ // 立即切换（推荐）
+ await this.switchVideoStream(videoFile, 'text', text);
+
+ // 或者等待当前视频播放完成再切换（可选）
+ // await this.waitForVideoToFinish();
+ // await this.switchVideoStream(videoFile, 'text', text);
+ } else {
+ // 直接切换
+ await this.switchVideoStream(videoFile, 'text', text);
+ }
// 通知服务器切换视频流
this.socket.emit('switch-video-stream', {
@@ -703,7 +733,7 @@ class WebRTCChat {
// 处理语音输入结果
async handleVoiceInput(text) {
// 根据文本查找对应视频
- let videoFile = this.videoMapping['默认'] || this.defaultVideo;
+ let videoFile = this.videoMapping['default'] || this.defaultVideo;
for (const [key, value] of Object.entries(this.videoMapping)) {
if (text.toLowerCase().includes(key.toLowerCase())) {
videoFile = value;
@@ -723,9 +753,12 @@ class WebRTCChat {
// 调用大模型处理
try {
- this.logMessage('正在处理语音输入，请稍候...', 'info');
- const result = await chatWithAudioStream(text);
- this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
+ this.logMessage('正在处理语音输入，请稍候...', 'info', this.currentVideoTag);
+ // if (this.currentVideoName === "default"){
+ const result = await chatWithAudioStream(text);
+ this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
+ // }
+
} catch (error) {
this.logMessage(`处理语音输入失败: ${error.message}`, 'error');
console.error('chatWithAudioStream error:', error);
diff --git a/src/minimaxi_stream.js b/src/minimaxi_stream.js
index a1eac7a..3311bf6 100644
--- a/src/minimaxi_stream.js
+++ b/src/minimaxi_stream.js
@@ -57,11 +57,24 @@ async function processAudioQueue() {
isProcessingQueue = true;
console.log('开始处理音频队列');
-
+ let isFirstChunk = true;
while (audioQueue.length > 0 || isPlaying) {
// 如果当前没有音频在播放，且队列中有音频
if (!isPlaying && audioQueue.length > 0) {
const audioItem = audioQueue.shift();
+ const sayName = 'say-5s-m-sw'
+ const targetVideo = '5.mp4'
+ // 如果是第一个音频片段，触发视频切换
+ if (sayName != window.webrtcApp.currentVideoTag && window.webrtcApp && window.webrtcApp.handleTextInput) {
+ try {
+ console.log('--------------触发视频切换:', sayName);
+ await window.webrtcApp.switchVideoStream(targetVideo, 'audio', 'say-5s-m-sw');
+ isFirstChunk = false;
+ window.webrtcApp.currentVideoTag = sayName;
+ } catch (error) {
+ console.error('视频切换失败:', error);
+ }
+ }
await playAudioData(audioItem.audioData);
} else {
// 等待一小段时间再检查
@@ -70,6 +83,10 @@ async function processAudioQueue() {
}
isProcessingQueue = false;
+ const text = 'default'
+ window.webrtcApp.currentVideoTag = text;
+ await window.webrtcApp.socket.emit('text-input', { text });
+ await window.webrtcApp.handleTextInput(text);
console.log('音频队列处理完成');
}
@@ -133,6 +150,11 @@ function getQueueStatus() {
async function requestMinimaxi({ apiKey, groupId, body, stream = true }) {
const url = `https://api.minimaxi.com/v1/t2a_v2`;
const reqBody = { ...body, stream };
+
+ // 添加这两行变量定义
+ let isFirstChunk = true;
+ // const currentText = body.text;
+
const response = await fetch(url, {
method: 'POST',
headers: {
@@ -197,11 +219,26 @@ async function requestMinimaxi({ apiKey, groupId, body, stream = true }) {
console.log('收到音频数据片段!', obj.data.audio.length);
audioHex += obj.data.audio;
+ // const sayName = 'say-5s-m-sw'
+ // // 如果是第一个音频片段，触发视频切换
+ // if (isFirstChunk && sayName != window.webrtcApp.currentVideoName && window.webrtcApp && window.webrtcApp.handleTextInput) {
+ // try {
+ // await window.webrtcApp.handleTextInput(sayName);
+ // isFirstChunk = false;
+ // window.webrtcApp.currentVideoName = sayName;
+ // } catch (error) {
+ // console.error('视频切换失败:', error);
+ // }
+ // }
+
// 立即播放这个音频片段
await playAudioChunk(obj.data.audio);
}
// status=2为最后一个chunk，记录完整结构
if (obj.data && obj.data.status === 2) {
+ // const text = 'default'
+ // await window.webrtcApp.socket.emit('text-input', { text });
+ // await window.webrtcApp.handleTextInput(text);
lastFullResult = obj;
console.log('收到最终状态');
}
diff --git a/videos/5.mp4 b/videos/5.mp4
new file mode 100644
index 0000000..de496e9
Binary files /dev/null and b/videos/5.mp4 differ
diff --git a/videos/6.mp4 b/videos/6.mp4
new file mode 100644
index 0000000..ea54411
Binary files /dev/null and b/videos/6.mp4 differ