new videos

chat_history.json

@@ -1 +1,272 @@
-[]
+[
+  {
+    "role": "user",
+    "content": "你好。",
+    "timestamp": "2025-07-27T14:31:39.908Z"
+  },
+  {
+    "role": "assistant",
+    "content": "哟，还这么客气呢？周末打算干啥呀？ ",
+    "timestamp": "2025-07-27T14:31:39.908Z"
+  },
+  {
+    "role": "user",
+    "content": "不打算干啥。",
+    "timestamp": "2025-07-27T14:31:52.513Z"
+  },
+  {
+    "role": "assistant",
+    "content": "得嘞，那咱宅家咋样，我发现个超搞笑综艺。 ",
+    "timestamp": "2025-07-27T14:31:52.513Z"
+  },
+  {
+    "role": "user",
+    "content": "哎，我不知道。",
+    "timestamp": "2025-07-27T14:32:00.781Z"
+  },
+  {
+    "role": "assistant",
+    "content": "不知道啥呀？综艺还是宅家咋安排？要不咱叫点烧烤边吃边看？ ",
+    "timestamp": "2025-07-27T14:32:00.781Z"
+  },
+  {
+    "role": "user",
+    "content": "到咯，佢到咯，我做饭啊。",
+    "timestamp": "2025-07-27T14:32:06.301Z"
+  },
+  {
+    "role": "assistant",
+    "content": "哟，背着我悄悄准备做饭啦！做啥好吃的，有我份不？ ",
+    "timestamp": "2025-07-27T14:32:06.301Z"
+  },
+  {
+    "role": "user",
+    "content": "到咯，佢到咯，我做饭啊。要3秒，要开大。",
+    "timestamp": "2025-07-27T14:32:08.465Z"
+  },
+  {
+    "role": "assistant",
+    "content": "啥跟啥啊？说的我一头雾水。你是准备做饭吗？需不需要我帮你弄点啥，冰箱里还有鸡蛋和青菜呢。 ",
+    "timestamp": "2025-07-27T14:32:08.465Z"
+  },
+  {
+    "role": "user",
+    "content": "Hello, hello.",
+    "timestamp": "2025-07-27T14:39:43.062Z"
+  },
+  {
+    "role": "assistant",
+    "content": "别搁那hello了，快跟我说说你做饭啥情况，要开啥大？是火开大点吗？ ",
+    "timestamp": "2025-07-27T14:39:43.062Z"
+  },
+  {
+    "role": "user",
+    "content": "你说话啊。",
+    "timestamp": "2025-07-27T14:40:45.074Z"
+  },
+  {
+    "role": "assistant",
+    "content": "我一直说呢！你做饭开啥大快说清楚，别搁这“挤牙膏”啦！ ",
+    "timestamp": "2025-07-27T14:40:45.074Z"
+  },
+  {
+    "role": "user",
+    "content": "挤牙膏。",
+    "timestamp": "2025-07-27T14:40:56.087Z"
+  },
+  {
+    "role": "assistant",
+    "content": "服了你这个老六，别老重复我话呀，好好跟我讲讲做饭到底咋回事。 ",
+    "timestamp": "2025-07-27T14:40:56.087Z"
+  },
+  {
+    "role": "user",
+    "content": "你好啊！",
+    "timestamp": "2025-07-27T14:44:25.948Z"
+  },
+  {
+    "role": "assistant",
+    "content": "别装礼貌啦，快接着说做饭那事儿，咋个“要3秒，要开大”法？ ",
+    "timestamp": "2025-07-27T14:44:25.948Z"
+  },
+  {
+    "role": "user",
+    "content": "开大招。",
+    "timestamp": "2025-07-27T14:46:10.186Z"
+  },
+  {
+    "role": "assistant",
+    "content": "靠！还开大招，你做饭当打游戏呢？说清楚要干啥！ ",
+    "timestamp": "2025-07-27T14:46:10.186Z"
+  },
+  {
+    "role": "user",
+    "content": "在干嘛呢？",
+    "timestamp": "2025-07-27T14:47:35.754Z"
+  },
+  {
+    "role": "assistant",
+    "content": "等你说清楚做饭的事儿呢！你到底要把啥开大，火还是调料量？ ",
+    "timestamp": "2025-07-27T14:47:35.754Z"
+  },
+  {
+    "role": "user",
+    "content": "火呀！",
+    "timestamp": "2025-07-27T14:47:47.933Z"
+  },
+  {
+    "role": "assistant",
+    "content": "妥，炒菜火大才香。炒啥呢？",
+    "timestamp": "2025-07-27T14:47:47.933Z"
+  },
+  {
+    "role": "user",
+    "content": "瞅你长得好看。",
+    "timestamp": "2025-07-27T14:48:00.643Z"
+  },
+  {
+    "role": "assistant",
+    "content": "少来转移话题啊，先把你炒的菜说清楚，我好给你参谋参谋这火咋开。 ",
+    "timestamp": "2025-07-27T14:48:00.643Z"
+  },
+  {
+    "role": "user",
+    "content": "我喜欢你。",
+    "timestamp": "2025-07-27T14:48:18.743Z"
+  },
+  {
+    "role": "assistant",
+    "content": "哟，嘴甜起来了。不过先别搁这哄我，菜还等着呢，到底炒啥？ ",
+    "timestamp": "2025-07-27T14:48:18.743Z"
+  },
+  {
+    "role": "user",
+    "content": "现在的孩子，真的没得到头，全部拿福祝了。",
+    "timestamp": "2025-07-27T14:48:32.883Z"
+  },
+  {
+    "role": "assistant",
+    "content": "啥意思啊？没太懂，先把你炒菜的事儿弄明白，这火还开着呢！ ",
+    "timestamp": "2025-07-27T14:48:32.883Z"
+  },
+  {
+    "role": "user",
+    "content": "那个",
+    "timestamp": "2025-07-27T14:48:45.683Z"
+  },
+  {
+    "role": "assistant",
+    "content": "别“那个”了，急死我，菜名快说！ ",
+    "timestamp": "2025-07-27T14:48:45.683Z"
+  },
+  {
+    "role": "user",
+    "content": "say-3s-m-sw",
+    "timestamp": "2025-07-27T14:53:33.356Z"
+  },
+  {
+    "role": "assistant",
+    "content": "这啥暗语啊？别整这些花里胡哨的，赶紧说炒啥菜，火大了糊锅啦！ ",
+    "timestamp": "2025-07-27T14:53:33.356Z"
+  },
+  {
+    "role": "user",
+    "content": "Hello.",
+    "timestamp": "2025-07-27T14:56:12.312Z"
+  },
+  {
+    "role": "assistant",
+    "content": "别搁这整洋文了，快告诉我炒啥，再不说菜没法要了！ ",
+    "timestamp": "2025-07-27T14:56:12.312Z"
+  },
+  {
+    "role": "user",
+    "content": "我喜欢你。",
+    "timestamp": "2025-07-27T14:56:26.151Z"
+  },
+  {
+    "role": "assistant",
+    "content": "知道啦知道啦，先解决炒菜这事儿，到底炒啥呀！ ",
+    "timestamp": "2025-07-27T14:56:26.151Z"
+  },
+  {
+    "role": "user",
+    "content": "也还行啊。",
+    "timestamp": "2025-07-27T14:56:38.392Z"
+  },
+  {
+    "role": "assistant",
+    "content": "啥也还行？你就痛痛快快说炒啥菜，别在这打哑谜了！ ",
+    "timestamp": "2025-07-27T14:56:38.392Z"
+  },
+  {
+    "role": "user",
+    "content": "你好啊！",
+    "timestamp": "2025-07-27T15:56:10.807Z"
+  },
+  {
+    "role": "assistant",
+    "content": "哟，整这么客气干啥，快把炒菜的事儿说明白。 ",
+    "timestamp": "2025-07-27T15:56:10.807Z"
+  },
+  {
+    "role": "user",
+    "content": "哈哈。",
+    "timestamp": "2025-07-27T15:56:20.531Z"
+  },
+  {
+    "role": "assistant",
+    "content": "别光笑啊，菜名呢？我都等半天了！ ",
+    "timestamp": "2025-07-27T15:56:20.531Z"
+  },
+  {
+    "role": "user",
+    "content": "是不会哦。",
+    "timestamp": "2025-07-27T15:56:22.209Z"
+  },
+  {
+    "role": "assistant",
+    "content": "啥不会？炒菜步骤不会还是菜名没定啊？别磨叽！ ",
+    "timestamp": "2025-07-27T15:56:22.210Z"
+  },
+  {
+    "role": "user",
+    "content": "你好。",
+    "timestamp": "2025-07-27T15:57:59.101Z"
+  },
+  {
+    "role": "assistant",
+    "content": "别老“你好”啦，咱快把炒菜这事儿搞定，你到底要炒啥！ ",
+    "timestamp": "2025-07-27T15:57:59.101Z"
+  },
+  {
+    "role": "user",
+    "content": "你好啊！",
+    "timestamp": "2025-07-27T16:07:56.242Z"
+  },
+  {
+    "role": "assistant",
+    "content": "靠！别老说这俩字，赶紧跟我讲菜，不然我要发火啦！ ",
+    "timestamp": "2025-07-27T16:07:56.242Z"
+  },
+  {
+    "role": "user",
+    "content": "吃什么菜？",
+    "timestamp": "2025-07-27T16:08:10.403Z"
+  },
+  {
+    "role": "assistant",
+    "content": "这就对了嘛！你可以试试炒个酸辣土豆丝，简单又好吃。你觉得咋样？ ",
+    "timestamp": "2025-07-27T16:08:10.403Z"
+  },
+  {
+    "role": "user",
+    "content": "我就是响应速度太长了。我没有问过他。我也不晓得。",
+    "timestamp": "2025-07-27T16:08:30.548Z"
+  },
+  {
+    "role": "assistant",
+    "content": "啥响应速度？越说越迷糊。你就说对酸辣土豆丝感不感兴趣嘛！ ",
+    "timestamp": "2025-07-27T16:08:30.548Z"
+  }
+]

server.js

@@ -90,7 +90,9 @@ const videoMapping = {
   'say-6s-m-e': '1-m.mp4',
   'default': '0.mp4',
   'say-5s-amplitude': '2.mp4',
-  'say-5s-m-e': 'zxc.mp4'
+  'say-5s-m-e': '4.mp4',
+  'say-5s-m-sw': '5.mp4',
+  'say-3s-m-sw': '6.mp4',
 };
 
 // 默认视频流配置
@@ -213,7 +215,7 @@ io.on('connection', (socket) => {
     console.log('收到文本输入:', text, '来自用户:', socket.id);
     
     // 根据文本查找对应视频
-    let videoFile = videoMapping['默认'];
+    let videoFile = videoMapping['default'];
     for (const [key, value] of Object.entries(videoMapping)) {
       if (text.toLowerCase().includes(key.toLowerCase())) {
         videoFile = value;
@@ -237,7 +239,7 @@ io.on('connection', (socket) => {
     console.log('收到语音输入:', text, '来自用户:', socket.id);
     
     // 根据语音识别的文本查找对应视频
-    let videoFile = videoMapping['默认'];
+    let videoFile = videoMapping['default'];
     for (const [key, value] of Object.entries(videoMapping)) {
       if (text.toLowerCase().includes(key.toLowerCase())) {
         videoFile = value;

src/audio_processor.js

@@ -93,7 +93,10 @@ class AudioProcessor {
             
             if (speechDuration >= this.minSpeechDuration) {
                 console.log(`语音结束，时长: ${speechDuration}ms`);
-                await this.processAudioBuffer();
+                console.log(window.webrtcApp.currentVideoTag)
+                if (window.webrtcApp.currentVideoTag==="default"){
+                    await this.processAudioBuffer();
+                }
                 this.onStatusUpdate('语音识别中...', 'processing');
             } else {
                 console.log('说话时长太短，忽略');

src/chat_with_audio.js

@@ -143,7 +143,7 @@ async function chatWithAudioStream(userInput) {
                     audioHex: audioResult.data.audio
                 });
                 console.log('音频已添加到队列，队列长度:', audioQueue.length);
-                
+                // this.socket.emit('audio-queue-updated', audioQueue.map(item => ({ text: item.text, hasAudio:!!item.audioHex })));
                 // 开始处理队列
                 processAudioQueue();
             }

src/index.html

@@ -37,9 +37,9 @@
             <div class="controls">
                 <button id="startButton" class="btn btn-primary">开始音频通话</button>
                 <button id="stopButton" class="btn btn-danger" disabled>停止通话</button>
-                <button id="muteButton" class="btn btn-secondary">静音</button>
+                <!-- <button id="muteButton" class="btn btn-secondary">静音</button>
                 <button id="defaultVideoButton" class="btn btn-info">回到默认视频</button>
-                <button id="testVideoButton" class="btn btn-warning">测试视频文件</button>
+                <button id="testVideoButton" class="btn btn-warning">测试视频文件</button> -->
             </div>
 
             <!-- 输入区域 -->
@@ -57,12 +57,12 @@
             </div>
 
             <!-- 视频选择 -->
-            <div class="video-selection">
+            <!-- <div class="video-selection">
                 <h3>选择要播放的视频</h3>
                 <div id="videoList" class="video-list">
-                    <!-- 视频列表将在这里动态生成 -->
+                    视频列表将在这里动态生成 -->
-                </div>
+                <!-- </div>
-            </div>
+            </div> -->
 
             <!-- 状态显示 -->
             <div class="status-section">

src/index.js

@@ -20,6 +20,7 @@ class WebRTCChat {
         this.audioChunks = [];
         this.videoMapping = {};
         this.defaultVideo = '0.mp4';
+        this.currentVideoTag = 'default';
         this.currentVideo = null;
         this.videoStreams = new Map(); // 存储不同视频的MediaStream
         this.currentVideoStream = null;
@@ -58,6 +59,8 @@ class WebRTCChat {
         this.loadVideoList();
         this.loadDefaultVideo();
         this.bindEvents();
+
+        window.webrtcApp = this;
     }
 
     initializeElements() {
@@ -77,7 +80,7 @@ class WebRTCChat {
         this.startVoiceButton = document.getElementById('startVoiceButton');
         this.stopVoiceButton = document.getElementById('stopVoiceButton');
         this.defaultVideoButton = document.getElementById('defaultVideoButton');
-        this.testVideoButton = document.getElementById('testVideoButton'); // 新增测试按钮
+        // this.testVideoButton = document.getElementById('testVideoButton'); // 新增测试按钮
         
         // 输入元素
         this.textInput = document.getElementById('textInput');
@@ -467,13 +470,13 @@ class WebRTCChat {
         this.stopButton.onclick = () => this.stopCall();
         
         // 静音按钮
-        this.muteButton.onclick = () => this.toggleMute();
+        // this.muteButton.onclick = () => this.toggleMute();
         
         // 回到默认视频按钮
-        this.defaultVideoButton.onclick = () => this.returnToDefaultVideo();
+        // this.defaultVideoButton.onclick = () => this.returnToDefaultVideo();
         
         // 测试视频文件按钮
-        this.testVideoButton.onclick = () => this.testAllVideoFiles();
+        // this.testVideoButton.onclick = () => this.testAllVideoFiles();
         
         // 发送文本按钮
         this.sendTextButton.onclick = () => this.sendText();
@@ -631,7 +634,8 @@ class WebRTCChat {
     }
 
     async sendText() {
-        const text = this.textInput.value.trim();
+        // const text = this.textInput.value.trim();
+        const text = 'say-3s-m-sw';
         if (text) {
             this.socket.emit('text-input', { text });
             this.logMessage(`发送文本: ${text}`, 'info');
@@ -643,8 +647,17 @@ class WebRTCChat {
                 const result = await chatWithAudioStream(text);
                 this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
                 
-                // 根据文本查找对应视频并切换
+                // // // 根据文本查找对应视频并切换
-                await this.handleTextInput(text);
+                // await this.handleTextInput(text);
+
+                // 并行执行两个操作
+                // const [result] = await Promise.all([
+                //     this.handleTextInput(text),
+                //     chatWithAudioStream(text)
+                //       // 视频切换可以立即开始
+                // ]);
+                
+                // this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
             } catch (error) {
                 this.logMessage(`处理文本失败: ${error.message}`, 'error');
                 console.error('chatWithAudioStream error:', error);
@@ -654,7 +667,7 @@ class WebRTCChat {
 
     async handleTextInput(text) {
         // 根据文本查找对应视频
-        let videoFile = this.videoMapping['默认'] || this.defaultVideo;
+        let videoFile = this.videoMapping['default'] || this.defaultVideo;
         for (const [key, value] of Object.entries(this.videoMapping)) {
             if (text.toLowerCase().includes(key.toLowerCase())) {
                 videoFile = value;
@@ -662,8 +675,25 @@ class WebRTCChat {
             }
         }
         
-        // 切换到对应的视频流
+            // 检查当前视频播放状态
-        await this.switchVideoStream(videoFile, 'text', text);
+        const currentVideo = this.recordedVideo;
+        const isVideoPlaying = !currentVideo.paused && !currentVideo.ended && currentVideo.currentTime > 0;
+        
+        if (isVideoPlaying && this.currentVideo !== videoFile) {
+            // 如果当前视频正在播放且需要切换到不同视频
+            // 可以选择立即切换或等待当前视频播放完成
+            console.log('当前视频正在播放，准备切换到:', videoFile);
+            
+            // 立即切换（推荐）
+            await this.switchVideoStream(videoFile, 'text', text);
+            
+            // 或者等待当前视频播放完成再切换（可选）
+            // await this.waitForVideoToFinish();
+            // await this.switchVideoStream(videoFile, 'text', text);
+        } else {
+            // 直接切换
+            await this.switchVideoStream(videoFile, 'text', text);
+        }
         
         // 通知服务器切换视频流
         this.socket.emit('switch-video-stream', { 
@@ -703,7 +733,7 @@ class WebRTCChat {
     // 处理语音输入结果
     async handleVoiceInput(text) {
         // 根据文本查找对应视频
-        let videoFile = this.videoMapping['默认'] || this.defaultVideo;
+        let videoFile = this.videoMapping['default'] || this.defaultVideo;
         for (const [key, value] of Object.entries(this.videoMapping)) {
             if (text.toLowerCase().includes(key.toLowerCase())) {
                 videoFile = value;
@@ -723,9 +753,12 @@ class WebRTCChat {
         
         // 调用大模型处理
         try {
-            this.logMessage('正在处理语音输入，请稍候...', 'info');
+            this.logMessage('正在处理语音输入，请稍候...', 'info', this.currentVideoTag);
-            const result = await chatWithAudioStream(text);
+            // if (this.currentVideoName === "default"){
-            this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
+                const result = await chatWithAudioStream(text);
+                this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
+            // }
+            
         } catch (error) {
             this.logMessage(`处理语音输入失败: ${error.message}`, 'error');
             console.error('chatWithAudioStream error:', error);

src/minimaxi_stream.js

@@ -57,11 +57,24 @@ async function processAudioQueue() {
   
   isProcessingQueue = true;
   console.log('开始处理音频队列');
-  
+  let isFirstChunk = true;
   while (audioQueue.length > 0 || isPlaying) {
     // 如果当前没有音频在播放，且队列中有音频
     if (!isPlaying && audioQueue.length > 0) {
       const audioItem = audioQueue.shift();
+      const sayName = 'say-5s-m-sw'
+      const targetVideo = '5.mp4'
+      // 如果是第一个音频片段，触发视频切换
+      if (sayName != window.webrtcApp.currentVideoTag && window.webrtcApp && window.webrtcApp.handleTextInput) {
+        try {
+          console.log('--------------触发视频切换:', sayName);
+          await window.webrtcApp.switchVideoStream(targetVideo, 'audio', 'say-5s-m-sw');
+          isFirstChunk = false;
+          window.webrtcApp.currentVideoTag = sayName;
+        } catch (error) {
+          console.error('视频切换失败:', error);
+        }
+      }
       await playAudioData(audioItem.audioData);
     } else {
       // 等待一小段时间再检查
@@ -70,6 +83,10 @@ async function processAudioQueue() {
   }
   
   isProcessingQueue = false;
+  const text = 'default'
+  window.webrtcApp.currentVideoTag = text;
+  await window.webrtcApp.socket.emit('text-input', { text });
+  await window.webrtcApp.handleTextInput(text);
   console.log('音频队列处理完成');
 }
 
@@ -133,6 +150,11 @@ function getQueueStatus() {
 async function requestMinimaxi({ apiKey, groupId, body, stream = true }) {
   const url = `https://api.minimaxi.com/v1/t2a_v2`;
   const reqBody = { ...body, stream };
+  
+  // 添加这两行变量定义
+  let isFirstChunk = true;
+  // const currentText = body.text;
+  
   const response = await fetch(url, {
     method: 'POST',
     headers: {
@@ -197,11 +219,26 @@ async function requestMinimaxi({ apiKey, groupId, body, stream = true }) {
                 console.log('收到音频数据片段!', obj.data.audio.length);
                 audioHex += obj.data.audio;
                 
+                // const sayName = 'say-5s-m-sw'
+                // // 如果是第一个音频片段，触发视频切换
+                // if (isFirstChunk && sayName != window.webrtcApp.currentVideoName && window.webrtcApp && window.webrtcApp.handleTextInput) {
+                //   try {
+                //     await window.webrtcApp.handleTextInput(sayName);
+                //     isFirstChunk = false;
+                //     window.webrtcApp.currentVideoName = sayName;
+                //   } catch (error) {
+                //     console.error('视频切换失败:', error);
+                //   }
+                // }
+                
                 // 立即播放这个音频片段
                 await playAudioChunk(obj.data.audio);
               }
               // status=2为最后一个chunk，记录完整结构
               if (obj.data && obj.data.status === 2) {
+                // const text = 'default'
+                // await window.webrtcApp.socket.emit('text-input', { text });
+                // await window.webrtcApp.handleTextInput(text);
                 lastFullResult = obj;
                 console.log('收到最终状态');
               }