console.log('视频文件:'); // WebRTC 音视频通话应用 // import { chatWithAudioStream } from './chat_with_audio.js'; import { chatWithAudioStream, initializeHistoryMessage } from './chat_with_audio.js'; import { AudioProcessor } from './audio_processor.js'; // 在应用初始化时调用 class WebRTCChat { constructor() { console.log('WebRTCChat 构造函数开始执行'); // 初始化历史消息(异步) this.initializeHistory(); this.socket = null; this.localStream = null; this.peerConnection = null; this.isRecording = false; this.mediaRecorder = null; this.audioChunks = []; this.videoMapping = {}; this.defaultVideo = 'chang.mp4'; this.interactionVideo = 'chang.mp4'; this.currentVideoTag = 'default'; this.currentVideo = null; this.videoStreams = new Map(); // 存储不同视频的MediaStream this.currentVideoStream = null; // 添加视频相关属性 this.videoSender = null; // WebRTC视频发送器 this.currentVideoStream = null; // 当前视频流 // 初始化音频处理器 console.log('开始初始化音频处理器'); // 初始化音频处理器 this.audioProcessor = new AudioProcessor({ onSpeechStart: () => { // this.voiceStatus.textContent = '检测到语音,开始录音...'; this.logMessage('检测到语音,开始录音...', 'info'); }, onSpeechEnd: () => { // 语音结束回调 }, onRecognitionResult: (text) => { // ASRTEXT = text; // this.voiceStatus.textContent = '识别完成'; this.logMessage(`语音识别结果: ${text}`, 'success'); this.handleVoiceInput(text); }, onError: (error) => { // this.voiceStatus.textContent = '识别失败'; this.logMessage(error, 'error'); }, onStatusUpdate: (message, status) => { // this.voiceStatus.textContent = message; } }); console.log('WebRTC 聊天应用初始化完成'); this.initializeElements(); this.initializeSocket(); this.loadVideoMapping(); this.loadVideoList(); this.loadDefaultVideo(); this.bindEvents(); // 在初始化完成后预加载常用视频 setTimeout(() => { this.logMessage('开始预加载常用视频...', 'info'); this.preloadCommonVideos().catch(error => { this.logMessage(`预加载过程出错: ${error.message}`, 'error'); }); }, 500); // 延迟2秒开始预加载,避免影响 window.webrtcApp = this; } initializeElements() { // 视频元素 this.localVideo = document.getElementById('localVideo'); this.remoteVideo = document.getElementById('remoteVideo'); this.recordedVideo = document.getElementById('recordedVideo'); this.recordedVideoBuffer = document.getElementById('recordedVideoBuffer'); // 新增缓冲视频元素 this.videoLoading = document.getElementById('videoLoading'); // 加载指示器 // 当前活跃的视频元素标识 this.activeVideoElement = 'main'; // 'main' 或 'buffer' // 音频状态元素 this.audioStatus = document.getElementById('audioStatus'); // 按钮元素 this.startButton = document.getElementById('startButton'); this.stopButton = document.getElementById('stopButton'); this.muteButton = document.getElementById('muteButton'); this.sendTextButton = document.getElementById('sendTextButton'); // this.startVoiceButton = document.getElementById('startVoiceButton'); // this.stopVoiceButton = document.getElementById('stopVoiceButton'); // this.defaultVideoButton = document.getElementById('defaultVideoButton'); // this.testVideoButton = document.getElementById('testVideoButton'); // 新增测试按钮 // 输入元素 this.textInput = document.getElementById('textInput'); // this.voiceStatus = document.getElementById('voiceStatus'); // 状态元素 this.connectionStatus = document.getElementById('connectionStatus'); this.messageLog = document.getElementById('messageLog'); this.currentVideoName = document.getElementById('currentVideoName'); this.videoList = document.getElementById('videoList'); } initializeSocket() { this.socket = io(); this.socket.on('connect', () => { this.updateStatus('已连接到服务器', 'connected'); this.logMessage('已连接到服务器', 'success'); }); this.socket.on('disconnect', () => { this.updateStatus('与服务器断开连接', 'disconnected'); this.logMessage('与服务器断开连接', 'error'); }); // WebRTC 信令处理 this.socket.on('offer', (data) => { this.handleOffer(data); }); this.socket.on('answer', (data) => { this.handleAnswer(data); }); this.socket.on('ice-candidate', (data) => { this.handleIceCandidate(data); }); // 视频流切换处理 this.socket.on('video-stream-switched', (data) => { this.logMessage(`收到视频流切换指令: ${data.videoFile} (${data.type}) 来自用户: ${data.from}`, 'info'); this.switchVideoStream(data.videoFile, data.type, data.text); }); // 通话开始处理 this.socket.on('call-started', (data) => { this.logMessage('通话已开始', 'success'); this.startDefaultVideoStream(); }); // 场景切换处理 this.socket.on('scene-switched', (data) => { this.logMessage(`场景已切换到: ${data.currentScene}`, 'info'); // 移除自动清除缓存和播放视频的逻辑 // 现在依赖页面刷新来处理缓存清除 }); } async initializeHistory() { try { await initializeHistoryMessage(); console.log('历史消息初始化完成'); } catch (error) { console.error('历史消息初始化失败:', error); } } async loadVideoMapping() { try { const response = await fetch('/api/video-mapping'); const data = await response.json(); this.videoMapping = data.mapping; this.interactionVideo = data.mapping['8-4-sh']; this.defaultVideo = data.mapping["default"]; this.logMessage('视频映射加载成功', 'success'); } catch (error) { this.logMessage('加载视频映射失败: ' + error.message, 'error'); } } async loadDefaultVideo() { try { const response = await fetch('/api/default-video'); const data = await response.json(); this.defaultVideo = data.defaultVideo; this.logMessage('默认视频配置加载成功', 'success'); } catch (error) { this.logMessage('加载默认视频配置失败: ' + error.message, 'error'); } } async loadVideoList() { try { const response = await fetch('/api/videos'); const data = await response.json(); this.renderVideoList(data.videos); this.logMessage('视频列表加载成功', 'success'); } catch (error) { this.logMessage('加载视频列表失败: ' + error.message, 'error'); } } renderVideoList(videos) { this.videoList.innerHTML = ''; videos.forEach(video => { const videoItem = document.createElement('div'); videoItem.className = 'video-item'; videoItem.textContent = video; videoItem.onclick = () => this.selectVideo(video); this.videoList.appendChild(videoItem); }); } selectVideo(videoFile) { // 移除之前的active类 document.querySelectorAll('.video-item').forEach(item => { item.classList.remove('active'); }); // 添加active类到选中的视频 event.target.classList.add('active'); // 切换到选中的视频流 this.switchVideoStream(videoFile, 'manual'); // 通知服务器切换视频流 this.socket.emit('switch-video-stream', { videoFile, type: 'manual' }); } async startDefaultVideoStream() { try { this.logMessage('开始创建默认视频流', 'info'); // 添加加载状态 this.recordedVideo.classList.add('loading'); // 创建默认视频的MediaStream const defaultStream = await this.createVideoStream(this.defaultVideo); // 等待流稳定 await new Promise(resolve => setTimeout(resolve, 500)); // 检查流是否有效 if (!defaultStream || defaultStream.getTracks().length === 0) { throw new Error('默认视频流创建失败'); } // 设置视频流 this.currentVideoStream = defaultStream; this.recordedVideo.srcObject = defaultStream; this.currentVideo = this.defaultVideo; this.currentVideoName.textContent = `默认视频: ${this.defaultVideo}`; // 等待视频元素准备就绪 await new Promise(resolve => { const checkReady = () => { if (this.recordedVideo.readyState >= 2) { // HAVE_CURRENT_DATA resolve(); } else { setTimeout(checkReady, 100); } }; checkReady(); }); // 确保视频开始播放 try { await this.recordedVideo.play(); this.logMessage('默认视频开始播放', 'success'); // 移除加载状态,添加播放状态 this.recordedVideo.classList.remove('loading'); this.recordedVideo.classList.add('playing'); } catch (playError) { this.logMessage(`默认视频播放失败: ${playError.message}`, 'error'); this.recordedVideo.classList.remove('loading'); } this.logMessage('默认视频流创建成功', 'success'); } catch (error) { this.logMessage('创建默认视频流失败: ' + error.message, 'error'); this.recordedVideo.classList.remove('loading'); } } async testVideoFile(videoFile) { return new Promise((resolve, reject) => { const testVideo = document.createElement('video'); testVideo.src = `/videos/${videoFile}`; testVideo.muted = true; testVideo.onloadedmetadata = () => { this.logMessage(`视频文件测试成功: ${videoFile} (${testVideo.videoWidth}x${testVideo.videoHeight})`, 'success'); resolve(true); }; testVideo.onerror = () => { this.logMessage(`视频文件测试失败: ${videoFile}`, 'error'); reject(new Error(`视频文件不存在或无法加载: ${videoFile}`)); }; // 设置超时 setTimeout(() => { reject(new Error(`视频文件加载超时: ${videoFile}`)); }, 10000); }); } async createVideoStream(videoFile) { // 检查缓存,但为每个视频创建独立的播放实例 const cacheKey = `${videoFile}_${Date.now()}`; // 添加时间戳确保唯一性 try { this.logMessage(`开始创建视频流: ${videoFile}`, 'info'); // 先测试视频文件是否存在 await this.testVideoFile(videoFile); // 创建video元素来加载视频 const video = document.createElement('video'); video.src = `/videos/${videoFile}`; video.muted = true; video.loop = true; video.autoplay = false; // 手动控制播放 video.crossOrigin = 'anonymous'; video.playsInline = true; // 预加载视频但不播放 video.preload = 'auto'; // 等待视频加载完成 await new Promise((resolve, reject) => { video.onloadeddata = () => { this.logMessage(`视频数据加载完成: ${videoFile}`, 'info'); // 确保从第一帧开始 video.currentTime = 0; resolve(); }; video.onerror = (error) => { this.logMessage(`视频加载失败: ${videoFile}`, 'error'); reject(error); }; }); // 创建MediaStream const canvas = document.createElement('canvas'); const ctx = canvas.getContext('2d'); // 设置canvas尺寸为视频尺寸 canvas.width = video.videoWidth || 640; canvas.height = video.videoHeight || 480; this.logMessage(`Canvas尺寸: ${canvas.width}x${canvas.height}`, 'info'); // 确保第一帧立即绘制,避免黑屏 await new Promise((resolve) => { const drawFirstFrame = () => { if (video.readyState >= video.HAVE_CURRENT_DATA) { ctx.drawImage(video, 0, 0, canvas.width, canvas.height); this.logMessage('已绘制第一帧到Canvas', 'info'); resolve(); } else { setTimeout(drawFirstFrame, 10); } }; drawFirstFrame(); }); // 开始播放视频 try { await video.play(); this.logMessage(`视频开始播放: ${videoFile}`, 'info'); } catch (playError) { this.logMessage(`视频播放失败: ${playError.message}`, 'error'); } // 等待视频真正开始播放 await new Promise(resolve => { const checkPlay = () => { if (video.readyState >= video.HAVE_CURRENT_DATA && !video.paused) { resolve(); } else { setTimeout(checkPlay, 50); } }; checkPlay(); }); // 绘制视频到canvas let lastDrawTime = 0; let isDrawing = false; const drawFrame = () => { const now = performance.now(); if (video.readyState >= video.HAVE_CURRENT_DATA && !isDrawing && (now - lastDrawTime > 16)) { isDrawing = true; lastDrawTime = now; ctx.drawImage(video, 0, 0, canvas.width, canvas.height); isDrawing = false; } requestAnimationFrame(drawFrame); }; // 开始绘制帧 drawFrame(); // 从canvas创建MediaStream const stream = canvas.captureStream(30); // 等待流稳定 await new Promise(resolve => { setTimeout(resolve, 200); // 减少等待时间 }); this.logMessage(`视频流创建成功: ${videoFile}`, 'success'); // 使用有限缓存策略(最多缓存3个视频流) if (this.videoStreams.size >= 3) { const firstKey = this.videoStreams.keys().next().value; const oldStream = this.videoStreams.get(firstKey); if (oldStream) { oldStream.getTracks().forEach(track => track.stop()); } this.videoStreams.delete(firstKey); } this.videoStreams.set(videoFile, stream); return stream; } catch (error) { this.logMessage(`创建视频流失败 ${videoFile}: ${error.message}`, 'error'); throw error; } } // 在应用初始化时预加载常用视频 async preloadCommonVideos() { // 获取所有可能需要的视频 console.log("default video, interaction video", [this.defaultVideo, this.interactionVideo]) const videosToPreload = new Set([this.defaultVideo, this.interactionVideo]); // 添加视频映射中的所有视频 // Object.values(this.videoMapping).forEach(video => { // videosToPreload.add(video); // }); // 特别确保添加了5.mp4(从日志看这是常用视频) // videosToPreload.add('d-0.mp4'); // 并行预加载,提高效率 const preloadPromises = Array.from(videosToPreload).map(async (videoFile) => { try { this.logMessage(`预加载视频: ${videoFile}`, 'info'); await this.createVideoStream(videoFile); this.logMessage(`预加载完成: ${videoFile}`, 'success'); } catch (error) { this.logMessage(`预加载失败: ${videoFile}: ${error.message}`, 'error'); } }); await Promise.allSettled(preloadPromises); } // async switchVideoStream(videoFile, type = '', text = '') { // try { // this.logMessage(`开始切换视频流: ${videoFile} (${type})`, 'info'); // // 检查是否已缓存 // const isCached = this.videoStreams.has(videoFile); // // 如果已缓存,直接使用,避免loading状态 // if (isCached) { // const cachedStream = this.videoStreams.get(videoFile); // if (cachedStream && cachedStream.getTracks().length > 0) { // // 直接切换到缓存的流 // this.currentVideoStream = cachedStream; // this.recordedVideo.srcObject = cachedStream; // this.currentVideo = videoFile; // // 立即播放,无需loading状态 // await this.recordedVideo.play(); // this.recordedVideo.classList.add('playing'); // this.logMessage(`使用缓存视频流: ${videoFile}`, 'success'); // return; // } // } // // 未缓存的视频才显示loading状态 // this.recordedVideo.classList.add('loading'); // // 先创建新的视频流 // const newStream = await this.createVideoStream(videoFile); // // 减少等待时间 // await new Promise(resolve => setTimeout(resolve, 100)); // // 检查流是否有效 // if (!newStream || newStream.getTracks().length === 0) { // throw new Error('创建的视频流无效'); // } // // 设置新的视频流 // this.currentVideoStream = newStream; // this.recordedVideo.srcObject = newStream; // this.currentVideo = videoFile; // // 确保视频开始播放 // try { // await this.recordedVideo.play(); // this.logMessage('视频元素开始播放', 'info'); // // 移除加载状态,添加播放状态 // this.recordedVideo.classList.remove('loading'); // this.recordedVideo.classList.add('playing'); // } catch (playError) { // this.logMessage(`视频播放失败: ${playError.message}`, 'error'); // this.recordedVideo.classList.remove('loading'); // } // // 现在停止旧的视频流 // if (this.currentVideoStream !== newStream) { // const oldStream = this.currentVideoStream; // setTimeout(() => { // if (oldStream) { // oldStream.getTracks().forEach(track => { // track.stop(); // this.logMessage(`已停止旧轨道: ${track.kind}`, 'info'); // }); // } // }, 1000); // 延迟1秒停止旧流,确保新流已经稳定 // } // if (text) { // this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`; // this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success'); // } else { // this.currentVideoName.textContent = `视频流: ${videoFile}`; // this.logMessage(`成功切换到视频流: ${videoFile}`, 'success'); // } // // 检查切换后的状态 // setTimeout(() => { // this.checkVideoStreamStatus(); // }, 1000); // } catch (error) { // this.logMessage(`切换视频流失败: ${error.message}`, 'error'); // this.recordedVideo.classList.remove('loading'); // // 如果切换失败,尝试回到默认视频 // if (videoFile !== this.defaultVideo) { // this.logMessage('尝试回到默认视频', 'info'); // await this.switchVideoStream(this.defaultVideo, 'fallback'); // } // } // } // 修改原有的switchVideoStream方法,使用新的平滑切换 async switchVideoStream(videoFile, type = '', text = '') { // 使用平滑切换方法 return await this.switchVideoStreamSmooth(videoFile, type, text); } // 使用replaceTrack方式切换视频 async switchVideoWithReplaceTrack(videoFile, type = '', text = '') { try { this.logMessage(`开始使用replaceTrack切换视频: ${videoFile}`, 'info'); // 创建新的视频流 const newVideoStream = await this.createVideoStream(videoFile); const newVideoTrack = newVideoStream.getVideoTracks()[0]; if (!newVideoTrack) { throw new Error('新视频流中没有视频轨道'); } // 如果有WebRTC连接且有视频发送器,使用replaceTrack if (this.peerConnection && this.videoSender) { await this.videoSender.replaceTrack(newVideoTrack); this.logMessage('WebRTC视频轨道替换成功', 'success'); } // 同时更新本地视频显示 if (this.recordedVideo) { // 停止当前视频流 if (this.currentVideoStream) { this.currentVideoStream.getTracks().forEach(track => track.stop()); } // 设置新的视频流 this.recordedVideo.srcObject = newVideoStream; this.currentVideoStream = newVideoStream; // 确保视频播放 try { await this.recordedVideo.play(); this.logMessage(`本地视频切换成功: ${videoFile}`, 'success'); } catch (playError) { this.logMessage(`本地视频播放失败: ${playError.message}`, 'error'); } } // 记录切换信息 if (type && text) { this.logMessage(`视频切换完成 - 类型: ${type}, 文本: ${text}`, 'info'); } return true; } catch (error) { this.logMessage(`replaceTrack视频切换失败: ${error.message}`, 'error'); console.error('switchVideoWithReplaceTrack error:', error); // 回退到原有的切换方式 try { await this.switchVideoStream(videoFile, type, text); this.logMessage('已回退到原有视频切换方式', 'info'); } catch (fallbackError) { this.logMessage(`回退切换也失败: ${fallbackError.message}`, 'error'); } return false; } } // 新增平滑视频切换方法 async switchVideoStreamSmooth(videoFile, type = '', text = '') { try { this.logMessage(`开始平滑切换视频流: ${videoFile} (${type})`, 'info'); // 检查是否已缓存,如果已缓存则不显示加载指示器 const isCached = this.videoStreams.has(videoFile); if (!isCached) { this.showVideoLoading(); } // 确定当前活跃的视频元素和缓冲元素 const currentVideo = this.activeVideoElement === 'main' ? this.recordedVideo : this.recordedVideoBuffer; const bufferVideo = this.activeVideoElement === 'main' ? this.recordedVideoBuffer : this.recordedVideo; // 检查是否已缓存 let newStream; if (isCached) { const cachedStream = this.videoStreams.get(videoFile); if (cachedStream && cachedStream.getTracks().length > 0) { newStream = cachedStream; this.logMessage(`使用缓存视频流: ${videoFile}`, 'success'); } else { newStream = await this.createVideoStream(videoFile); } } else { newStream = await this.createVideoStream(videoFile); } // 在缓冲视频元素中预加载新视频 bufferVideo.srcObject = newStream; // 等待缓冲视频准备就绪 await new Promise((resolve, reject) => { const timeout = setTimeout(() => { reject(new Error('视频加载超时')); }, 5000); const onCanPlay = () => { clearTimeout(timeout); bufferVideo.removeEventListener('canplay', onCanPlay); bufferVideo.removeEventListener('error', onError); resolve(); }; const onError = (error) => { clearTimeout(timeout); bufferVideo.removeEventListener('canplay', onCanPlay); bufferVideo.removeEventListener('error', onError); reject(error); }; bufferVideo.addEventListener('canplay', onCanPlay); bufferVideo.addEventListener('error', onError); // 开始播放缓冲视频 bufferVideo.play().catch(onError); }); // 只有显示了加载指示器才隐藏 if (!isCached) { this.hideVideoLoading(); } // 执行淡入淡出切换 // await this.performVideoTransition(currentVideo, bufferVideo); // 更新当前视频流和活跃元素 this.currentVideoStream = newStream; this.currentVideo = videoFile; this.activeVideoElement = this.activeVideoElement === 'main' ? 'buffer' : 'main'; // 停止旧视频流(延迟停止避免闪烁) setTimeout(() => { if (currentVideo.srcObject && currentVideo.srcObject !== newStream) { currentVideo.srcObject.getTracks().forEach(track => track.stop()); currentVideo.srcObject = null; } }, 1000); // 更新WebRTC连接中的视频轨道 if (this.peerConnection && this.videoSender) { const newVideoTrack = newStream.getVideoTracks()[0]; if (newVideoTrack) { await this.videoSender.replaceTrack(newVideoTrack); } } // 更新显示信息 if (text) { this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`; this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success'); } else { this.currentVideoName.textContent = `视频流: ${videoFile}`; this.logMessage(`成功切换到视频流: ${videoFile}`, 'success'); } } catch (error) { this.logMessage(`平滑切换视频流失败: ${error.message}`, 'error'); // 确保隐藏加载指示器 this.hideVideoLoading(); // 如果切换失败,尝试回到默认视频 if (videoFile !== this.defaultVideo) { this.logMessage('尝试回到默认视频', 'info'); await this.switchVideoStreamSmooth(this.defaultVideo, 'fallback'); } } } // 执行视频过渡动画 async performVideoTransition(currentVideo, bufferVideo) { return new Promise((resolve) => { // 添加切换类 currentVideo.classList.add('switching'); bufferVideo.classList.add('switching'); // 等待CSS过渡完成 setTimeout(() => { // 移除切换类 currentVideo.classList.remove('switching'); bufferVideo.classList.remove('switching'); // 交换z-index if (currentVideo.style.zIndex === '2' || !currentVideo.style.zIndex) { currentVideo.style.zIndex = '1'; bufferVideo.style.zIndex = '2'; } else { currentVideo.style.zIndex = '2'; bufferVideo.style.zIndex = '1'; } resolve(); }, 500); // 与CSS过渡时间一致 }); } // 显示加载指示器 showVideoLoading() { if (this.videoLoading) { this.videoLoading.classList.add('show'); } } // 隐藏加载指示器 hideVideoLoading() { if (this.videoLoading) { this.videoLoading.classList.remove('show'); } } bindEvents() { // 开始通话按钮 this.startButton.onclick = () => this.startCall(); // 停止通话按钮 this.stopButton.onclick = () => this.stopCall(); // 静音按钮 // this.muteButton.onclick = () => this.toggleMute(); // 回到默认视频按钮 // this.defaultVideoButton.onclick = () => this.returnToDefaultVideo(); // 测试视频文件按钮 // this.testVideoButton.onclick = () => this.testAllVideoFiles(); // 发送文本按钮 this.sendTextButton.onclick = () => this.sendText(); // 回车键发送文本 this.textInput.onkeypress = (e) => { if (e.key === 'Enter') { this.sendText(); } }; // 语音输入按钮 // this.startVoiceButton.onclick = () => this.startVoiceRecording(); // this.stopVoiceButton.onclick = () => this.stopVoiceRecording(); } async startCall() { try { // 切换到通话中图标 this.switchToCallingIcon(); // 添加更详细的错误处理 console.log('开始请求麦克风权限...'); this.localStream = await navigator.mediaDevices.getUserMedia({ video: false, audio: true }); console.log('麦克风权限获取成功'); await this.createPeerConnection(); await this.startVoiceRecording(); this.startButton.disabled = true; this.stopButton.disabled = false; // 显示结束通话按钮 this.stopButton.classList.add('show'); this.updateAudioStatus('已连接', 'connected'); this.logMessage('音频通话已开始', 'success'); // 确保视频映射已加载 if (Object.keys(this.videoMapping).length === 0) { await this.loadVideoMapping(); } this.logMessage(`视频映射已加载: ${Object.keys(this.videoMapping).length} 个映射`, 'info'); // 通知服务器通话开始 this.socket.emit('call-started'); // 开始播放当前场景的默认视频 await this.startDefaultVideoStream(); } catch (error) { // 如果出错,恢复到默认图标 this.switchToDefaultIcon(); this.logMessage('无法访问麦克风: ' + error.message, 'error'); } } stopCall() { // 恢复到默认图标 this.switchToDefaultIcon(); // 发送用户关闭连接事件到后端 if (this.socket && this.socket.connected) { this.socket.emit('user-disconnect'); } // 停止音频处理器 if (this.audioProcessor) { this.audioProcessor.stopRecording(); } if (this.localStream) { this.localStream.getTracks().forEach(track => { track.stop(); console.log(`停止轨道: ${track.kind}`); }); this.localStream = null; } if (this.peerConnection) { this.peerConnection.close(); this.peerConnection = null; } // 直接刷新页面清除所有缓存 console.log('通话已结束,正在刷新页面清除缓存...'); window.location.reload(); } // 清除视频缓存的方法 clearVideoCache() { // 清除视频元素缓存 if (this.recordedVideo) { this.recordedVideo.src = ''; this.recordedVideo.srcObject = null; this.recordedVideo.load(); // 重新加载空视频 // 暂停视频播放 this.recordedVideo.pause(); } if (this.recordedVideoBuffer) { this.recordedVideoBuffer.src = ''; this.recordedVideoBuffer.srcObject = null; this.recordedVideoBuffer.load(); this.recordedVideoBuffer.pause(); } // 清除视频流缓存 if (this.currentVideoStream) { this.currentVideoStream.getTracks().forEach(track => track.stop()); this.currentVideoStream = null; } // 清除视频流映射缓存 this.videoStreams.clear(); // 重置视频相关状态 this.currentVideo = null; this.activeVideoElement = 'main'; this.videoSender = null; this.logMessage('视频缓存已清除,视频已停止', 'info'); } // 添加清除API缓存的方法 async clearApiCache() { // 重新加载视频映射 this.videoMapping = {}; await this.loadVideoMapping(); // 重新加载默认视频 await this.loadDefaultVideo(); // 重新加载视频列表 await this.loadVideoList(); this.logMessage('API缓存已清除并重新加载', 'info'); } async createPeerConnection() { const configuration = { iceServers: [ { urls: "stun:stun.qq.com:3478" }, { urls: "stun:stun.miwifi.com:3478" }, { urls: 'stun:stun.l.google.com:19302' }, { urls: 'stun:stun1.l.google.com:19302' } ] }; this.peerConnection = new RTCPeerConnection(configuration); // 添加本地音频流 this.localStream.getTracks().forEach(track => { this.peerConnection.addTrack(track, this.localStream); }); // 添加初始视频流(默认视频) try { const initialVideoStream = await this.createVideoStream(this.defaultVideo); const videoTrack = initialVideoStream.getVideoTracks()[0]; if (videoTrack) { this.videoSender = this.peerConnection.addTrack(videoTrack, initialVideoStream); this.currentVideoStream = initialVideoStream; this.logMessage('初始视频轨道已添加到WebRTC连接', 'success'); } } catch (error) { this.logMessage(`添加初始视频轨道失败: ${error.message}`, 'error'); } // 处理远程流 this.peerConnection.ontrack = (event) => { this.remoteVideo.srcObject = event.streams[0]; }; // 处理ICE候选 this.peerConnection.onicecandidate = (event) => { if (event.candidate) { this.socket.emit('ice-candidate', { candidate: event.candidate }); } }; // 创建并发送offer this.peerConnection.createOffer() .then(offer => this.peerConnection.setLocalDescription(offer)) .then(() => { this.socket.emit('offer', { offer: this.peerConnection.localDescription }); }) .catch(error => { this.logMessage('创建offer失败: ' + error.message, 'error'); }); } async handleOffer(data) { if (!this.peerConnection) { await this.startCall(); } await this.peerConnection.setRemoteDescription(new RTCSessionDescription(data.offer)); const answer = await this.peerConnection.createAnswer(); await this.peerConnection.setLocalDescription(answer); this.socket.emit('answer', { answer: this.peerConnection.localDescription }); } async handleAnswer(data) { if (this.peerConnection) { await this.peerConnection.setRemoteDescription(new RTCSessionDescription(data.answer)); } } async handleIceCandidate(data) { if (this.peerConnection) { await this.peerConnection.addIceCandidate(new RTCIceCandidate(data.candidate)); } } toggleMute() { if (this.localStream) { const audioTrack = this.localStream.getAudioTracks()[0]; if (audioTrack) { audioTrack.enabled = !audioTrack.enabled; this.muteButton.textContent = audioTrack.enabled ? '静音' : '取消静音'; this.logMessage(audioTrack.enabled ? '已取消静音' : '已静音', 'info'); } } } async sendText() { const text = this.textInput.value.trim(); // const text = 'say-5s-m-sw'; if (text) { this.socket.emit('text-input', { text }); this.logMessage(`发送文本: ${text}`, 'info'); this.textInput.value = ''; try { // 调用chat_with_audio进行大模型回答和音频合成 this.logMessage('正在处理文本,请稍候...', 'info'); const result = await chatWithAudioStream(text); this.logMessage(`大模型回答: ${result.llmResponse}`, 'success'); // // // 根据文本查找对应视频并切换 // await this.handleTextInput(text); // 并行执行两个操作 // const [result] = await Promise.all([ // this.handleTextInput(text), // chatWithAudioStream(text) // // 视频切换可以立即开始 // ]); // this.logMessage(`大模型回答: ${result.llmResponse}`, 'success'); } catch (error) { this.logMessage(`处理文本失败: ${error.message}`, 'error'); console.error('chatWithAudioStream error:', error); } } } async handleTextInput(text) { // 根据文本查找对应视频 let videoFile = this.videoMapping['default'] || this.defaultVideo; for (const [key, value] of Object.entries(this.videoMapping)) { if (text.toLowerCase().includes(key.toLowerCase())) { videoFile = value; break; } } // 检查当前视频播放状态 const currentVideo = this.recordedVideo; const isVideoPlaying = !currentVideo.paused && !currentVideo.ended && currentVideo.currentTime > 0; if (isVideoPlaying && this.currentVideo !== videoFile) { // 如果当前视频正在播放且需要切换到不同视频 // 可以选择立即切换或等待当前视频播放完成 console.log('当前视频正在播放,准备切换到:', videoFile); // 立即切换(推荐) await this.switchVideoStream(videoFile, 'text', text); // 或者等待当前视频播放完成再切换(可选) // await this.waitForVideoToFinish(); // await this.switchVideoStream(videoFile, 'text', text); } else { // 直接切换 await this.switchVideoStream(videoFile, 'text', text); } // 通知服务器切换视频流 this.socket.emit('switch-video-stream', { videoFile, type: 'text', text }); } // 修改:使用音频处理器的语音录制功能 async startVoiceRecording() { const success = await this.audioProcessor.startRecording(this.localStream); if (success) { this.logMessage('高级语音录制已启动', 'success'); } else { this.logMessage('录音启动失败', 'error'); } } // 修改:停止语音录制 stopVoiceRecording() { this.audioProcessor.stopRecording(); // this.startVoiceButton.disabled = false; // this.stopVoiceButton.disabled = true; // this.startVoiceButton.classList.remove('recording'); // this.voiceStatus.textContent = '点击开始语音输入'; this.logMessage('语音录制已停止', 'info'); } // 处理语音输入结果 async handleVoiceInput(text) { if(text == ""){ console.log("识别到用户未说话") return } // 根据文本查找对应视频 let videoFile = this.videoMapping['default'] || this.defaultVideo; for (const [key, value] of Object.entries(this.videoMapping)) { if (text.toLowerCase().includes(key.toLowerCase())) { videoFile = value; break; } } // 切换到对应的视频流 await this.switchVideoStream(videoFile, 'voice', text); // 通知服务器切换视频流 this.socket.emit('switch-video-stream', { videoFile, type: 'voice', text }); // 调用大模型处理 try { this.logMessage('正在处理语音输入,请稍候...', 'info', this.currentVideoTag); // if (this.currentVideoName === "default"){ const result = await chatWithAudioStream(text); this.logMessage(`大模型回答: ${result.llmResponse}`, 'success'); // } } catch (error) { this.logMessage(`处理语音输入失败: ${error.message}`, 'error'); console.error('chatWithAudioStream error:', error); } } // 删除原有的简单音频处理方法 // processVoiceInput() 和 simulateSpeechRecognition() 方法已被移除 simulateSpeechRecognition() { // 模拟语音识别,随机返回预设的文本 const texts = ['你好', '再见', '谢谢', 'hello', 'goodbye', 'thank you']; return texts[Math.floor(Math.random() * texts.length)]; } returnToDefaultVideo() { this.switchVideoStream(this.defaultVideo, 'default'); this.socket.emit('return-to-default'); this.logMessage(`已返回至默认视频: ${this.defaultVideo}`, 'info'); } updateStatus(message, type) { this.connectionStatus.textContent = message; this.connectionStatus.className = `status ${type}`; } updateAudioStatus(message, type) { this.audioStatus.textContent = message; this.audioStatus.className = `status-indicator ${type}`; } logMessage(message, type = 'info') { const logEntry = document.createElement('div'); logEntry.className = type; logEntry.textContent = `[${new Date().toLocaleTimeString()}] ${message}`; this.messageLog.appendChild(logEntry); this.messageLog.scrollTop = this.messageLog.scrollHeight; } checkVideoStreamStatus() { const status = { hasStream: !!this.currentVideoStream, streamTracks: this.currentVideoStream ? this.currentVideoStream.getTracks().length : 0, videoReadyState: this.recordedVideo.readyState, videoPaused: this.recordedVideo.paused, videoCurrentTime: this.recordedVideo.currentTime, videoDuration: this.recordedVideo.duration, currentVideo: this.currentVideo }; this.logMessage(`视频流状态: ${JSON.stringify(status)}`, 'info'); return status; } // 添加图标切换方法 switchToCallingIcon() { const callIcon = document.getElementById('callIcon'); const callingIcon = document.getElementById('callingIcon'); const startButton = this.startButton; if (callIcon && callingIcon && startButton) { callIcon.style.display = 'none'; callingIcon.style.display = 'block'; startButton.classList.add('calling'); startButton.title = '通话中...'; } } switchToDefaultIcon() { const callIcon = document.getElementById('callIcon'); const callingIcon = document.getElementById('callingIcon'); const startButton = this.startButton; if (callIcon && callingIcon && startButton) { callIcon.style.display = 'block'; callingIcon.style.display = 'none'; startButton.classList.remove('calling'); startButton.title = '开始通话'; startButton.disabled = false; } } async testAllVideoFiles() { this.logMessage('开始测试所有视频文件...', 'info'); const videoFiles = ['asd.mp4', 'zxc.mp4', 'jkl.mp4']; for (const videoFile of videoFiles) { try { await this.testVideoFile(videoFile); } catch (error) { this.logMessage(`视频文件 ${videoFile} 测试失败: ${error.message}`, 'error'); } } this.logMessage('视频文件测试完成', 'info'); // 检查当前视频流状态 this.checkVideoStreamStatus(); } } // 页面加载完成后初始化应用 document.addEventListener('DOMContentLoaded', () => { console.log('DOMContentLoaded 事件触发'); try { new WebRTCChat(); } catch (error) { console.error('WebRTCChat 初始化失败:', error); } });