console.log('视频文件:'); // WebRTC 音视频通话应用 // import { chatWithAudioStream } from './chat_with_audio.js'; import { chatWithAudioStream, initializeHistoryMessage } from './chat_with_audio.js'; import { AudioProcessor } from './audio_processor.js'; // 在应用初始化时调用 class WebRTCChat { constructor() { console.log('WebRTCChat 构造函数开始执行'); // 初始化历史消息(异步) this.initializeHistory(); this.socket = null; this.localStream = null; this.peerConnection = null; this.isRecording = false; this.mediaRecorder = null; this.audioChunks = []; this.videoMapping = {}; this.defaultVideo = 'chang.mp4'; this.interactionVideo = 'chang.mp4'; this.currentVideoTag = 'default'; this.currentVideo = null; this.videoStreams = new Map(); // 存储不同视频的MediaStream this.currentVideoStream = null; this.precreatedStreams = new Map(); // 预创建的视频流 this.importantVideos = ['d-3s.mp4', 's-1.mp4']; // 重要视频列表 this.isInitialized = false; // 添加视频加载状态标志 this.isVideoReady = false; this.isDefaultVideoLoaded = false; this.retryCount = 0; // 添加重试计数器 // 添加视频相关属性 this.videoSender = null; // WebRTC视频发送器 this.currentVideoStream = null; // 当前视频流 // 初始化音频处理器 console.log('开始初始化音频处理器'); // 初始化音频处理器 this.audioProcessor = new AudioProcessor({ onSpeechStart: () => { // this.voiceStatus.textContent = '检测到语音,开始录音...'; this.logMessage('检测到语音,开始录音...', 'info'); }, onSpeechEnd: () => { // 语音结束回调 }, onRecognitionResult: (text) => { // ASRTEXT = text; // this.voiceStatus.textContent = '识别完成'; this.logMessage(`语音识别结果: ${text}`, 'success'); this.handleVoiceInput(text); }, onError: (error) => { // this.voiceStatus.textContent = '识别失败'; this.logMessage(error, 'error'); }, onStatusUpdate: (message, status) => { // this.voiceStatus.textContent = message; } }); console.log('WebRTC 聊天应用初始化完成'); this.initializeSocket(); this.initializeElements(); this.loadVideoMapping(); this.loadVideoList(); // 只预加载视频资源,不显示视频 this.preloadVideoResources(); this.bindEvents(); // 在初始化完成后预加载常用视频 // setTimeout(() => { // this.logMessage('开始预加载常用视频...', 'info'); // this.preloadCommonVideos().catch(error => { // this.logMessage(`预加载过程出错: ${error.message}`, 'error'); // }); // }, 500); // 预创建重要视频流 setTimeout(() => { this.precreateImportantVideos().catch(error => { this.logMessage(`预创建重要视频流失败: ${error.message}`, 'error'); }); }, 1000); window.webrtcApp = this; } // 新增方法:预加载视频资源 async preloadVideoResources() { try { await this.loadDefaultVideo(); // 预创建视频流但不设置到video元素 const defaultStream = await this.createVideoStreamOptimized(this.defaultVideo); this.precreatedStreams.set(this.defaultVideo, defaultStream); this.isVideoReady = true; this.isDefaultVideoLoaded = true; // 启用开始通话按钮 // if (this.startButton) { // this.startButton.disabled = false; // this.startButton.style.opacity = '1'; // } console.log('视频资源预加载完成,可以开始通话', 'success'); } catch (error) { this.logMessage(`视频资源预加载失败: ${error.message}`, 'error'); this.isVideoReady = false; } } initializeElements() { // 视频元素 this.localVideo = document.getElementById('localVideo'); this.remoteVideo = document.getElementById('remoteVideo'); this.recordedVideo = document.getElementById('recordedVideo'); this.recordedVideoBuffer = document.getElementById('recordedVideoBuffer'); // 新增缓冲视频元素 this.videoLoading = document.getElementById('videoLoading'); // 加载指示器 // 头像和视频容器元素 this.avatarContainer = document.getElementById('avatarContainer'); this.videoContainer = document.getElementById('videoContainer'); // 当前活跃的视频元素标识 this.activeVideoElement = 'main'; // 'main' 或 'buffer' // 音频状态元素 this.audioStatus = document.getElementById('audioStatus'); // 按钮元素 this.startButton = document.getElementById('startButton'); this.stopButton = document.getElementById('stopButton'); this.muteButton = document.getElementById('muteButton'); this.sendTextButton = document.getElementById('sendTextButton'); // 初始状态下禁用开始通话按钮,直到视频加载完成 if (this.startButton) { this.startButton.disabled = true; this.startButton.style.opacity = '0.5'; // this.startButton.title = '加载中,请稍候...'; } // this.startVoiceButton = document.getElementById('startVoiceButton'); // this.stopVoiceButton = document.getElementById('stopVoiceButton'); // this.defaultVideoButton = document.getElementById('defaultVideoButton'); // this.testVideoButton = document.getElementById('testVideoButton'); // 新增测试按钮 // 输入元素 this.textInput = document.getElementById('textInput'); // this.voiceStatus = document.getElementById('voiceStatus'); // 状态元素 this.connectionStatus = document.getElementById('connectionStatus'); this.messageLog = document.getElementById('messageLog'); this.currentVideoName = document.getElementById('currentVideoName'); this.videoList = document.getElementById('videoList'); // 等待连接提示元素 this.connectionWaiting = document.getElementById('connectionWaiting'); } initializeSocket() { this.socket = io(); this.socket.on('connect', () => { this.updateStatus('已连接到服务器', 'connected'); this.logMessage('已连接到服务器', 'success'); }); this.socket.on('disconnect', () => { this.connectionStatus.style.display = 'none'; this.logMessage('与服务器断开连接', 'error'); }); // WebRTC 信令处理 this.socket.on('offer', (data) => { this.handleOffer(data); }); this.socket.on('answer', (data) => { this.handleAnswer(data); }); this.socket.on('ice-candidate', (data) => { this.handleIceCandidate(data); }); // 视频流切换处理 this.socket.on('video-stream-switched', (data) => { this.logMessage(`收到视频流切换指令: ${data.videoFile} (${data.type}) 来自用户: ${data.from}`, 'info'); this.switchVideoStream(data.videoFile, data.type, data.text); }); // 通话开始处理 this.socket.on('call-started', (data) => { console.log('通话已开始', 'success'); // 移除这里的视频流启动,因为现在在startCall中处理 }); // 场景切换处理 this.socket.on('scene-switched', (data) => { this.logMessage(`场景已切换到: ${data.scene.name}`, 'info'); // 更新视频映射 this.videoMapping = data.mapping; this.interactionVideo = data.mapping.interactionVideo; this.defaultVideo = data.mapping.defaultVideo; // 立即切换到新场景的默认视频 this.switchVideoStream(this.defaultVideo, 'scene-change'); console.log('场景切换完成,新的视频配置:', { defaultVideo: this.defaultVideo, interactionVideo: this.interactionVideo, tag: data.mapping.tag }); }); } async initializeHistory() { try { await initializeHistoryMessage(); console.log('历史消息初始化完成'); } catch (error) { console.error('历史消息初始化失败:', error); } } async loadVideoMapping() { try { const response = await fetch('/api/video-mapping'); const data = await response.json(); this.videoMapping = data.mapping; this.interactionVideo = data.mapping['8-4-sh']; this.defaultVideo = data.mapping["default"]; console.log('映射加载成功', 'success'); } catch (error) { this.logMessage('加载视频映射失败: ' + error.message, 'error'); } } async loadDefaultVideo() { try { const response = await fetch('/api/default-video'); const data = await response.json(); this.defaultVideo = data.defaultVideo; this.logMessage('默认视频配置加载成功', 'success'); } catch (error) { this.logMessage('加载默认视频配置失败: ' + error.message, 'error'); } } async loadVideoList() { try { const response = await fetch('/api/videos'); const data = await response.json(); this.renderVideoList(data.videos); this.logMessage('视频列表加载成功', 'success'); } catch (error) { this.logMessage('加载视频列表失败: ' + error.message, 'error'); } } renderVideoList(videos) { this.videoList.innerHTML = ''; videos.forEach(video => { const videoItem = document.createElement('div'); videoItem.className = 'video-item'; videoItem.textContent = video; videoItem.onclick = () => this.selectVideo(video); this.videoList.appendChild(videoItem); }); } selectVideo(videoFile) { // 移除之前的active类 document.querySelectorAll('.video-item').forEach(item => { item.classList.remove('active'); }); // 添加active类到选中的视频 event.target.classList.add('active'); // 切换到选中的视频流 this.switchVideoStream(videoFile, 'manual'); // 通知服务器切换视频流 this.socket.emit('switch-video-stream', { videoFile, type: 'manual' }); } async startDefaultVideoStream() { try { console.log('开始创建默认视频流', 'info'); // 显示视频元素 if (this.recordedVideo) { this.recordedVideo.style.display = 'block'; } // 添加加载状态 this.recordedVideo.classList.add('loading'); // 创建默认视频的MediaStream let defaultStream = this.precreatedStreams.get(this.defaultVideo); // 检查流是否有效,如果无效则重新创建 if (!defaultStream || defaultStream.getTracks().length === 0 || defaultStream.getTracks().some(track => track.readyState === 'ended')) { console.log('预创建流无效,重新创建默认视频流'); try { defaultStream = await this.createVideoStreamOptimized(this.defaultVideo); this.precreatedStreams.set(this.defaultVideo, defaultStream); } catch (createError) { throw new Error(`重新创建默认视频流失败: ${createError.message}`); } } // 等待流稳定 await new Promise(resolve => setTimeout(resolve, 1000)); // 再次检查流是否有效 if (!defaultStream || defaultStream.getTracks().length === 0) { throw new Error('默认视频流创建失败'); } // 设置视频流 this.currentVideoStream = defaultStream; this.recordedVideo.srcObject = defaultStream; this.recordedVideoBuffer.srcObject = this.precreatedStreams.get(this.interactionVideo); // this.recordedVideoBuffer.style.zIndex = "10" // this.recordedVideoBuffer.style.opacity = "1"; // 添加这行 // this.recordedVideo.style.zIndex = "-1" // this.recordedVideo.style.opacity = "0"; // this.recordedVideoBuffer.play(); this.currentVideo = this.defaultVideo; this.currentVideoName.textContent = `默认视频: ${this.defaultVideo}`; // 等待视频元素准备就绪 await new Promise(resolve => { const checkReady = () => { if (this.recordedVideo.readyState >= 2) { // HAVE_CURRENT_DATA resolve(); } else { setTimeout(checkReady, 100); } }; checkReady(); }); // 确保视频开始播放 try { await this.recordedVideo.play(); // this.logMessage('默认视频开始播放', 'success'); // 移除加载状态,添加播放状态 this.recordedVideo.classList.remove('loading'); this.recordedVideo.classList.add('playing'); } catch (playError) { // this.logMessage(`默认视频播放失败: ${playError.message}`, 'error'); this.recordedVideo.classList.remove('loading'); } // 再次检查视频是否准备就绪 // this.startButton.disabled = false; console.log('默认流创建成功', 'success'); } catch (error) { console.log('创建默认视频流失败: ' + error.message, 'error'); // 隐藏视频元素 if (this.recordedVideo) { this.recordedVideo.style.display = 'none'; } this.recordedVideo.classList.remove('loading'); // 添加重试机制 if (!this.retryCount) this.retryCount = 0; if (this.retryCount < 2) { this.retryCount++; console.log(`尝试重新创建默认视频流 (${this.retryCount}/2)`); setTimeout(() => this.startDefaultVideoStream(), 1000); } else { this.retryCount = 0; console.log('默认视频流创建失败,已达到最大重试次数', 'error'); } } } async testVideoFile(videoFile) { return new Promise((resolve, reject) => { const testVideo = document.createElement('video'); testVideo.src = `/videos/${videoFile}`; testVideo.muted = true; testVideo.onloadedmetadata = () => { this.logMessage(`视频文件测试成功: ${videoFile} (${testVideo.videoWidth}x${testVideo.videoHeight})`, 'success'); resolve(true); }; testVideo.onerror = () => { this.logMessage(`视频文件测试失败: ${videoFile}`, 'error'); reject(new Error(`视频文件不存在或无法加载: ${videoFile}`)); }; // 设置超时 setTimeout(() => { reject(new Error(`视频文件加载超时: ${videoFile}`)); }, 10000); }); } // 预创建重要视频流 async precreateImportantVideos() { if (this.isInitialized) return; console.log('开始预创建重要流...', 'info'); for (const videoFile of [this.interactionVideo, this.defaultVideo]) { try { const stream = await this.createVideoStreamOptimized(videoFile); this.precreatedStreams.set(videoFile, stream); this.logMessage(`预创建视频流成功: ${videoFile}`, 'success'); } catch (error) { this.logMessage(`预创建视频流失败: ${videoFile} - ${error.message}`, 'error'); } } // 启用开始通话按钮 if (this.startButton) { this.startButton.disabled = false; this.startButton.style.opacity = '1'; } this.isInitialized = true; this.logMessage('重要视频流预创建完成', 'success'); } // 优化的视频流创建方法 async createVideoStreamOptimized(videoFile) { try { // 先测试视频文件是否存在 await this.testVideoFile(videoFile); // 创建video元素 const video = document.createElement('video'); video.src = `/videos/${videoFile}`; video.muted = true; video.loop = true; video.autoplay = false; video.crossOrigin = 'anonymous'; video.playsInline = true; video.preload = 'auto'; // 等待视频加载 await new Promise((resolve, reject) => { video.onloadeddata = () => { video.currentTime = 0; resolve(); }; video.onerror = reject; }); // 创建优化的Canvas const canvas = document.createElement('canvas'); const ctx = canvas.getContext('2d', { alpha: false, // 禁用透明度以提高性能 willReadFrequently: false // 优化Canvas性能 }); canvas.width = video.videoWidth || 640; canvas.height = video.videoHeight || 480; // 绘制第一帧 await new Promise((resolve) => { const drawFirstFrame = () => { if (video.readyState >= video.HAVE_CURRENT_DATA) { ctx.drawImage(video, 0, 0, canvas.width, canvas.height); resolve(); } else { setTimeout(drawFirstFrame, 10); } }; drawFirstFrame(); }); // 启动视频播放 await video.play(); // 优化的帧绘制 - 使用更高效的节流 let animationId; let lastFrameTime = 0; const targetFPS = 30; const frameInterval = 1000 / targetFPS; const drawFrame = (currentTime) => { if (currentTime - lastFrameTime >= frameInterval) { if (video.readyState >= video.HAVE_CURRENT_DATA) { ctx.drawImage(video, 0, 0, canvas.width, canvas.height); } lastFrameTime = currentTime; } animationId = requestAnimationFrame(drawFrame); }; animationId = requestAnimationFrame(drawFrame); // 创建MediaStream const stream = canvas.captureStream(30); // 存储清理函数 stream._cleanup = () => { if (animationId) { cancelAnimationFrame(animationId); } video.pause(); video.src = ''; video.load(); }; return stream; } catch (error) { this.logMessage(`优化视频流创建失败 ${videoFile}: ${error.message}`, 'error'); throw error; } } async createVideoStream(videoFile) { // 检查缓存,但为每个视频创建独立的播放实例 const cacheKey = `${videoFile}_${Date.now()}`; // 添加时间戳确保唯一性 try { this.logMessage(`开始创建视频流: ${videoFile}`, 'info'); // 先测试视频文件是否存在 await this.testVideoFile(videoFile); // 创建video元素来加载视频 const video = document.createElement('video'); video.src = `/videos/${videoFile}`; video.muted = true; video.loop = true; video.autoplay = false; // 手动控制播放 video.crossOrigin = 'anonymous'; video.playsInline = true; // 预加载视频但不播放 video.preload = 'auto'; // 等待视频加载完成 await new Promise((resolve, reject) => { video.onloadeddata = () => { this.logMessage(`视频数据加载完成: ${videoFile}`, 'info'); // 确保从第一帧开始 video.currentTime = 0; resolve(); }; video.onerror = (error) => { this.logMessage(`视频加载失败: ${videoFile}`, 'error'); reject(error); }; }); // 创建MediaStream const canvas = document.createElement('canvas'); const ctx = canvas.getContext('2d'); // 设置canvas尺寸为视频尺寸 canvas.width = video.videoWidth || 640; canvas.height = video.videoHeight || 480; this.logMessage(`Canvas尺寸: ${canvas.width}x${canvas.height}`, 'info'); // 确保第一帧立即绘制,避免黑屏 await new Promise((resolve) => { const drawFirstFrame = () => { if (video.readyState >= video.HAVE_CURRENT_DATA) { ctx.drawImage(video, 0, 0, canvas.width, canvas.height); this.logMessage('已绘制第一帧到Canvas', 'info'); resolve(); } else { setTimeout(drawFirstFrame, 10); } }; drawFirstFrame(); }); // 开始播放视频 try { await video.play(); this.logMessage(`视频开始播放: ${videoFile}`, 'info'); } catch (playError) { this.logMessage(`视频播放失败: ${playError.message}`, 'error'); } // 等待视频真正开始播放 await new Promise(resolve => { const checkPlay = () => { if (video.readyState >= video.HAVE_CURRENT_DATA && !video.paused) { resolve(); } else { setTimeout(checkPlay, 50); } }; checkPlay(); }); // 绘制视频到canvas let lastDrawTime = 0; let isDrawing = false; const drawFrame = () => { const now = performance.now(); if (video.readyState >= video.HAVE_CURRENT_DATA && !isDrawing && (now - lastDrawTime > 16)) { isDrawing = true; lastDrawTime = now; ctx.drawImage(video, 0, 0, canvas.width, canvas.height); isDrawing = false; } requestAnimationFrame(drawFrame); }; // 开始绘制帧 drawFrame(); // 从canvas创建MediaStream const stream = canvas.captureStream(30); // 等待流稳定 await new Promise(resolve => { setTimeout(resolve, 200); // 减少等待时间 }); this.logMessage(`视频流创建成功: ${videoFile}`, 'success'); // 使用有限缓存策略(最多缓存3个视频流) if (this.videoStreams.size >= 3) { const firstKey = this.videoStreams.keys().next().value; const oldStream = this.videoStreams.get(firstKey); if (oldStream) { oldStream.getTracks().forEach(track => track.stop()); } this.videoStreams.delete(firstKey); } this.videoStreams.set(videoFile, stream); return stream; } catch (error) { this.logMessage(`创建视频流失败 ${videoFile}: ${error.message}`, 'error'); throw error; } } // 在应用初始化时预加载常用视频 async preloadCommonVideos() { // 获取所有可能需要的视频 console.log("default video, interaction video", [this.defaultVideo, this.interactionVideo]) const videosToPreload = new Set([this.defaultVideo, this.interactionVideo]); // 添加视频映射中的所有视频 // Object.values(this.videoMapping).forEach(video => { // videosToPreload.add(video); // }); // 特别确保添加了5.mp4(从日志看这是常用视频) // videosToPreload.add('d-0.mp4'); // 并行预加载,提高效率 const preloadPromises = Array.from(videosToPreload).map(async (videoFile) => { try { this.logMessage(`预加载视频: ${videoFile}`, 'info'); await this.createVideoStream(videoFile); this.logMessage(`预加载完成: ${videoFile}`, 'success'); } catch (error) { this.logMessage(`预加载失败: ${videoFile}: ${error.message}`, 'error'); } }); await Promise.allSettled(preloadPromises); } // async switchVideoStream(videoFile, type = '', text = '') { // try { // this.logMessage(`开始切换视频流: ${videoFile} (${type})`, 'info'); // // 检查是否已缓存 // const isCached = this.videoStreams.has(videoFile); // // 如果已缓存,直接使用,避免loading状态 // if (isCached) { // const cachedStream = this.videoStreams.get(videoFile); // if (cachedStream && cachedStream.getTracks().length > 0) { // // 直接切换到缓存的流 // this.currentVideoStream = cachedStream; // this.recordedVideo.srcObject = cachedStream; // this.currentVideo = videoFile; // // 立即播放,无需loading状态 // await this.recordedVideo.play(); // this.recordedVideo.classList.add('playing'); // this.logMessage(`使用缓存视频流: ${videoFile}`, 'success'); // return; // } // } // // 未缓存的视频才显示loading状态 // this.recordedVideo.classList.add('loading'); // // 先创建新的视频流 // const newStream = await this.createVideoStream(videoFile); // // 减少等待时间 // await new Promise(resolve => setTimeout(resolve, 100)); // // 检查流是否有效 // if (!newStream || newStream.getTracks().length === 0) { // throw new Error('创建的视频流无效'); // } // // 设置新的视频流 // this.currentVideoStream = newStream; // this.recordedVideo.srcObject = newStream; // this.currentVideo = videoFile; // // 确保视频开始播放 // try { // await this.recordedVideo.play(); // this.logMessage('视频元素开始播放', 'info'); // // 移除加载状态,添加播放状态 // this.recordedVideo.classList.remove('loading'); // this.recordedVideo.classList.add('playing'); // } catch (playError) { // this.logMessage(`视频播放失败: ${playError.message}`, 'error'); // this.recordedVideo.classList.remove('loading'); // } // // 现在停止旧的视频流 // if (this.currentVideoStream !== newStream) { // const oldStream = this.currentVideoStream; // setTimeout(() => { // if (oldStream) { // oldStream.getTracks().forEach(track => { // track.stop(); // this.logMessage(`已停止旧轨道: ${track.kind}`, 'info'); // }); // } // }, 1000); // 延迟1秒停止旧流,确保新流已经稳定 // } // if (text) { // this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`; // this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success'); // } else { // this.currentVideoName.textContent = `视频流: ${videoFile}`; // this.logMessage(`成功切换到视频流: ${videoFile}`, 'success'); // } // // 检查切换后的状态 // setTimeout(() => { // this.checkVideoStreamStatus(); // }, 1000); // } catch (error) { // this.logMessage(`切换视频流失败: ${error.message}`, 'error'); // this.recordedVideo.classList.remove('loading'); // // 如果切换失败,尝试回到默认视频 // if (videoFile !== this.defaultVideo) { // this.logMessage('尝试回到默认视频', 'info'); // await this.switchVideoStream(this.defaultVideo, 'fallback'); // } // } // } // 修改视频切换方法,直接使用预加载视频切换,不使用WebRTC传输 async switchVideoStream(videoFile, type = '', text = '') { if (this.interactionVideo === videoFile) { // 确保缓冲视频已经准备好并且可见 // this.recordedVideoBuffer.style.opacity = "1"; // 使用 zIndex 层叠,新视频在上层 this.recordedVideoBuffer.style.zIndex = "2"; this.recordedVideo.style.zIndex = "1"; // 延迟隐藏下层视频,确保无缝切换 // setTimeout(() => { // this.recordedVideo.style.opacity = "0"; // }, 100); } else { // 确保主视频已经准备好并且可见 // this.recordedVideo.style.opacity = "1"; // 使用 zIndex 层叠,主视频在上层 this.recordedVideo.style.zIndex = "2"; this.recordedVideoBuffer.style.zIndex = "1"; // 延迟隐藏下层视频,确保无缝切换 // setTimeout(() => { // this.recordedVideoBuffer.style.opacity = "0"; // }, 100); } } // 添加视频帧缓存方法 captureVideoFrame(videoElement) { const canvas = document.createElement('canvas'); const ctx = canvas.getContext('2d'); canvas.width = videoElement.videoWidth || videoElement.clientWidth; canvas.height = videoElement.videoHeight || videoElement.clientHeight; // 绘制当前视频帧到canvas ctx.drawImage(videoElement, 0, 0, canvas.width, canvas.height); return canvas.toDataURL('image/jpeg', 0.8); } // 创建静态帧显示元素 createFrameOverlay(frameData) { const overlay = document.createElement('div'); overlay.style.position = 'absolute'; overlay.style.top = '0'; overlay.style.left = '0'; overlay.style.width = '100%'; overlay.style.height = '100%'; overlay.style.backgroundImage = `url(${frameData})`; overlay.style.backgroundSize = 'cover'; overlay.style.backgroundPosition = 'center'; overlay.style.backgroundRepeat = 'no-repeat'; overlay.style.zIndex = '10'; overlay.style.pointerEvents = 'none'; overlay.style.willChange = 'auto'; // 优化渲染性能 overlay.id = 'video-frame-overlay'; return overlay; } // 新增平滑视频切换方法 - 等待完全传输后切换 async switchVideoStreamSmooth(videoFile, type = '', text = '') { try { this.logMessage(`开始等待视频流传输完成: ${videoFile} (${type})`, 'info'); // 1. 获取当前和缓冲视频元素 const currentVideo = this.activeVideoElement === 'main' ? this.recordedVideo : this.recordedVideoBuffer; const bufferVideo = this.activeVideoElement === 'main' ? this.recordedVideoBuffer : this.recordedVideo; // 2. 检查是否已缓存 const isCached = this.videoStreams.has(videoFile); if (!isCached) { this.showVideoLoading(); } // 3. 准备新视频流 let newStream; if (isCached) { const cachedStream = this.videoStreams.get(videoFile); if (cachedStream && cachedStream.getTracks().length > 0) { newStream = cachedStream; this.logMessage(`使用缓存视频流: ${videoFile}`, 'success'); } else { newStream = await this.createVideoStream(videoFile); } } else { newStream = await this.createVideoStream(videoFile); } // 4. 在缓冲视频元素中预加载新视频 bufferVideo.srcObject = newStream; // 5. 等待视频流完全传输和准备就绪 await new Promise((resolve, reject) => { const timeout = setTimeout(() => { reject(new Error('视频流传输超时')); }, 15000); // 增加超时时间到15秒,确保有足够时间传输 let frameCount = 0; const minFrames = 10; // 增加最小帧数确保传输完整 let isStreamReady = false; const onReady = () => { clearTimeout(timeout); bufferVideo.removeEventListener('canplay', onCanPlay); bufferVideo.removeEventListener('canplaythrough', onCanPlayThrough); bufferVideo.removeEventListener('timeupdate', onTimeUpdate); bufferVideo.removeEventListener('error', onError); bufferVideo.removeEventListener('loadeddata', onLoadedData); resolve(); }; // 检查视频数据是否完全加载 const onLoadedData = () => { this.logMessage(`视频数据开始加载: ${videoFile}`, 'info'); }; // 检查视频是否可以流畅播放(数据传输充足) const onCanPlayThrough = () => { this.logMessage(`视频流传输充足,可以流畅播放: ${videoFile}`, 'success'); isStreamReady = true; // 开始播放并等待稳定帧 bufferVideo.play().then(() => { const checkFrameStability = () => { if (bufferVideo.currentTime > 0) { frameCount++; if (frameCount >= minFrames && isStreamReady) { // 额外等待确保传输稳定 setTimeout(() => { onReady(); }, 200); // 等待200ms确保传输稳定 } else { requestAnimationFrame(checkFrameStability); } } else { requestAnimationFrame(checkFrameStability); } }; requestAnimationFrame(checkFrameStability); }).catch(reject); }; // 基本播放准备就绪 const onCanPlay = () => { if (!isStreamReady) { this.logMessage(`视频基本准备就绪,等待完全传输: ${videoFile}`, 'info'); // 如果还没有canplaythrough事件,继续等待 } }; // 时间更新监听(备用检查) const onTimeUpdate = () => { if (bufferVideo.currentTime > 0 && isStreamReady) { frameCount++; if (frameCount >= minFrames) { setTimeout(() => { onReady(); }, 200); } } }; const onError = (error) => { clearTimeout(timeout); bufferVideo.removeEventListener('canplay', onCanPlay); bufferVideo.removeEventListener('canplaythrough', onCanPlayThrough); bufferVideo.removeEventListener('timeupdate', onTimeUpdate); bufferVideo.removeEventListener('error', onError); bufferVideo.removeEventListener('loadeddata', onLoadedData); reject(error); }; // 检查当前状态 if (bufferVideo.readyState >= 4) { // HAVE_ENOUGH_DATA onCanPlayThrough(); } else { bufferVideo.addEventListener('loadeddata', onLoadedData); bufferVideo.addEventListener('canplay', onCanPlay); bufferVideo.addEventListener('canplaythrough', onCanPlayThrough); bufferVideo.addEventListener('timeupdate', onTimeUpdate); bufferVideo.addEventListener('error', onError); } }); // 6. 最终确认视频流完全准备就绪 await new Promise(resolve => { let confirmCount = 0; const maxConfirms = 3; const finalCheck = () => { if (bufferVideo.readyState >= 4 && bufferVideo.currentTime > 0) { confirmCount++; if (confirmCount >= maxConfirms) { this.logMessage(`视频流传输完成,准备切换: ${videoFile}`, 'success'); resolve(); } else { setTimeout(finalCheck, 50); // 每50ms检查一次 } } else { setTimeout(finalCheck, 50); } }; finalCheck(); }); // 7. 执行无缝切换(当前视频继续播放直到新视频完全准备好) bufferVideo.style.zIndex = '2'; currentVideo.style.zIndex = '1'; this.logMessage(`视频切换完成: ${videoFile}`, 'success'); // 8. 隐藏加载指示器 if (!isCached) { this.hideVideoLoading(); } // 9. 更新状态 this.currentVideoStream = newStream; this.currentVideo = videoFile; this.activeVideoElement = this.activeVideoElement === 'main' ? 'buffer' : 'main'; // 10. 延迟清理旧视频流 setTimeout(() => { if (currentVideo.srcObject && currentVideo.srcObject !== newStream) { currentVideo.srcObject.getTracks().forEach(track => track.stop()); currentVideo.srcObject = null; } }, 1000); // 11. 更新WebRTC连接 if (this.peerConnection && this.videoSender) { const newVideoTrack = newStream.getVideoTracks()[0]; if (newVideoTrack) { await this.videoSender.replaceTrack(newVideoTrack); } } // 12. 更新显示信息 if (text) { this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`; this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success'); } else { this.currentVideoName.textContent = `视频流: ${videoFile}`; this.logMessage(`成功切换到视频流: ${videoFile}`, 'success'); } } catch (error) { this.logMessage(`视频流切换失败: ${error.message}`, 'error'); this.hideVideoLoading(); // 回退到默认视频 if (videoFile !== this.defaultVideo) { this.logMessage('尝试回到默认视频', 'info'); await this.switchVideoStreamSmooth(this.defaultVideo, 'fallback'); } } } // 显示加载指示器 showVideoLoading() { if (this.videoLoading) { this.videoLoading.classList.add('show'); } } // 隐藏加载指示器 hideVideoLoading() { if (this.videoLoading) { this.videoLoading.classList.remove('show'); } } bindEvents() { // 开始通话按钮 - 添加视频准备状态检查 this.startButton.onclick = () => { if (!this.isVideoReady) { this.logMessage('还在加载中,请稍候...', 'warning'); return; } this.startCall(); }; // 停止通话按钮 - 改为调用 userDisconnect this.stopButton.onclick = () => this.userDisconnect(); // 静音按钮 // this.muteButton.onclick = () => this.toggleMute(); // 回到默认视频按钮 // this.defaultVideoButton.onclick = () => this.returnToDefaultVideo(); // 测试视频文件按钮 // this.testVideoButton.onclick = () => this.testAllVideoFiles(); // 发送文本按钮 this.sendTextButton.onclick = () => this.sendText(); // 回车键发送文本 this.textInput.onkeypress = (e) => { if (e.key === 'Enter') { this.sendText(); } }; // 语音输入按钮 // this.startVoiceButton.onclick = () => this.startVoiceRecording(); // this.stopVoiceButton.onclick = () => this.stopVoiceRecording(); } // 显示等待连接提示 showConnectionWaiting() { if (this.connectionWaiting) { this.connectionWaiting.style.display = 'block'; // 使用setTimeout确保display设置后再添加show类 setTimeout(() => { this.connectionWaiting.classList.add('show'); }, 10); } } // 隐藏等待连接提示 hideConnectionWaiting() { if (this.connectionWaiting) { this.connectionWaiting.classList.remove('show'); // 等待动画完成后隐藏元素 setTimeout(() => { this.connectionWaiting.style.display = 'none'; }, 300); } } async startCall() { try { // 检查所有必要条件 if (!this.isVideoReady || !this.isDefaultVideoLoaded) { this.logMessage('视频资源尚未准备就绪,请稍候...', 'warning'); return; } if (!this.socket || !this.socket.connected) { this.logMessage('网络连接未就绪,请稍候...', 'warning'); return; } // 隐藏开始通话按钮 this.startButton.style.display = 'none'; // 显示等待连接提示 this.showConnectionWaiting(); // 切换到通话中图标 this.switchToCallingIcon(); // 现在才开始显示视频 await this.startDefaultVideoStream(); // 添加更详细的错误处理 console.log('开始请求麦克风权限...'); this.localStream = await navigator.mediaDevices.getUserMedia({ video: false, audio: true }); console.log('麦克风权限获取成功'); await this.createPeerConnection(); await this.startVoiceRecording(); this.startButton.disabled = true; this.startButton.style.opacity = '0.5' this.stopButton.disabled = false; // 显示结束通话按钮 this.stopButton.style.display = 'block'; // 隐藏头像,显示视频 if (this.videoContainer) { this.videoContainer.classList.add('calling'); } this.updateAudioStatus('已连接', 'connected'); this.logMessage('音频通话已开始', 'success'); // 确保视频映射已加载 if (Object.keys(this.videoMapping).length === 0) { await this.loadVideoMapping(); } this.logMessage(`视频映射已加载: ${Object.keys(this.videoMapping).length} 个映射`, 'info'); // 通知服务器通话开始 this.socket.emit('call-started'); // 开始播放当前场景的默认视频 // await this.precreateImportantVideos(); // 隐藏等待连接提示 this.hideConnectionWaiting(); } catch (error) { this.logMessage(`开始通话失败: ${error.message}`, 'error'); // 恢复开始通话按钮 this.startButton.style.display = 'block'; // 如果出错,隐藏等待连接提示并恢复到默认图标 this.hideConnectionWaiting(); this.switchToDefaultIcon(); } } stopCall() { // 隐藏等待连接提示 this.hideConnectionWaiting(); // 恢复到默认图标 this.switchToDefaultIcon(); // 只有用户主动点击关闭通话时才发送断开事件 // 移除自动发送 user-disconnect 事件 // if (this.socket && this.socket.connected) { // this.socket.emit('user-disconnect'); // } // 停止音频处理器 if (this.audioProcessor) { this.audioProcessor.stopRecording(); } if (this.localStream) { this.localStream.getTracks().forEach(track => { track.stop(); console.log(`停止轨道: ${track.kind}`); }); this.localStream = null; } if (this.peerConnection) { this.peerConnection.close(); this.peerConnection = null; } // 隐藏结束通话按钮 this.stopButton.style.display = 'none'; this.stopButton.disabled = true; // 隐藏视频元素 if (this.recordedVideo) { this.recordedVideo.style.display = 'none'; this.recordedVideo.srcObject = null; } if (this.recordedVideoBuffer) { this.recordedVideoBuffer.style.display = 'none'; this.recordedVideoBuffer.srcObject = null; } // 显示开始通话按钮 this.startButton.disabled = true; this.startButton.style.display = 'block'; this.startButton.style.opacity = '0.5'; // 移除页面刷新,保持websocket连接 // setTimeout(() => { // window.location.reload(); // }, 300); // 清理视频缓存和预创建流 this.clearVideoCache(); setTimeout(() => { // 显示头像,隐藏视频 if (this.videoContainer) { this.videoContainer.classList.remove('calling'); } // 重新初始化重要视频流 this.precreateImportantVideos().then(() => { // 重新启动默认视频流 this.startDefaultVideoStream(); }); }, 300); } // 新增:用户主动断开连接的方法 userDisconnect() { // 发送用户关闭连接事件到后端 if (this.socket && this.socket.connected) { this.socket.emit('user-disconnect'); // 等待服务器确认断开后再刷新 this.socket.on('disconnect', () => { console.log('WebSocket已断开,准备刷新页面...'); setTimeout(() => { window.location.reload(); }, 500); }); // 主动断开连接 setTimeout(() => { this.socket.disconnect(); }, 200); // 兜底机制:如果2秒内没有正常断开,强制刷新 setTimeout(() => { if (this.socket && this.socket.connected) { console.log('WebSocket断开超时,强制刷新页面'); window.location.reload(); } }, 2000); } else { // 如果socket已经断开,直接刷新 setTimeout(() => { window.location.reload(); }, 100); } // 调用停止通话 this.stopCall(); } // 清除视频缓存的方法 clearVideoCache() { // 清除视频元素缓存 if (this.recordedVideo) { this.recordedVideo.src = ''; this.recordedVideo.srcObject = null; this.recordedVideo.load(); // 重新加载空视频 // 暂停视频播放 this.recordedVideo.pause(); } if (this.recordedVideoBuffer) { this.recordedVideoBuffer.src = ''; this.recordedVideoBuffer.srcObject = null; this.recordedVideoBuffer.load(); this.recordedVideoBuffer.pause(); } // 清除视频流缓存 if (this.currentVideoStream) { this.currentVideoStream.getTracks().forEach(track => track.stop()); this.currentVideoStream = null; } // 清除视频流映射缓存 this.videoStreams.clear(); // 清除预创建的视频流 if (this.precreatedStreams) { this.precreatedStreams.forEach((stream, videoFile) => { if (stream) { stream.getTracks().forEach(track => track.stop()); } }); this.precreatedStreams.clear(); } // 重置初始化状态,允许重新预创建 this.isInitialized = false; // 重置视频相关状态 this.currentVideo = null; this.activeVideoElement = 'main'; this.videoSender = null; this.logMessage('视频缓存已清除,视频已停止', 'info'); } // 添加清除API缓存的方法 async clearApiCache() { // 重新加载视频映射 this.videoMapping = {}; await this.loadVideoMapping(); // 重新加载默认视频 await this.loadDefaultVideo(); // 重新加载视频列表 await this.loadVideoList(); this.logMessage('API缓存已清除并重新加载', 'info'); } async createPeerConnection() { const configuration = { iceServers: [ { urls: "stun:stun.qq.com:3478" }, { urls: "stun:stun.miwifi.com:3478" }, { urls: 'stun:stun.l.google.com:19302' }, { urls: 'stun:stun1.l.google.com:19302' } ] }; this.peerConnection = new RTCPeerConnection(configuration); // 添加本地音频流 this.localStream.getTracks().forEach(track => { this.peerConnection.addTrack(track, this.localStream); }); // 移除视频轨道添加逻辑,只使用音频进行WebRTC通信 // 视频将直接在本地切换,不通过WebRTC传输 // 处理远程流 this.peerConnection.ontrack = (event) => { this.remoteVideo.srcObject = event.streams[0]; }; // 处理ICE候选 this.peerConnection.onicecandidate = (event) => { if (event.candidate) { this.socket.emit('ice-candidate', { candidate: event.candidate }); } }; // 创建并发送offer this.peerConnection.createOffer() .then(offer => this.peerConnection.setLocalDescription(offer)) .then(() => { this.socket.emit('offer', { offer: this.peerConnection.localDescription }); }) .catch(error => { this.logMessage('创建offer失败: ' + error.message, 'error'); }); } async handleOffer(data) { if (!this.peerConnection) { await this.startCall(); } await this.peerConnection.setRemoteDescription(new RTCSessionDescription(data.offer)); const answer = await this.peerConnection.createAnswer(); await this.peerConnection.setLocalDescription(answer); this.socket.emit('answer', { answer: this.peerConnection.localDescription }); } async handleAnswer(data) { if (this.peerConnection) { await this.peerConnection.setRemoteDescription(new RTCSessionDescription(data.answer)); } } async handleIceCandidate(data) { if (this.peerConnection) { await this.peerConnection.addIceCandidate(new RTCIceCandidate(data.candidate)); } } toggleMute() { if (this.localStream) { const audioTrack = this.localStream.getAudioTracks()[0]; if (audioTrack) { audioTrack.enabled = !audioTrack.enabled; this.muteButton.textContent = audioTrack.enabled ? '静音' : '取消静音'; this.logMessage(audioTrack.enabled ? '已取消静音' : '已静音', 'info'); } } } async sendText() { const text = this.textInput.value.trim(); // const text = 'say-5s-m-sw'; if (text) { this.socket.emit('text-input', { text }); this.logMessage(`发送文本: ${text}`, 'info'); this.textInput.value = ''; try { // 调用chat_with_audio进行大模型回答和音频合成 this.logMessage('正在处理文本,请稍候...', 'info'); const result = await chatWithAudioStream(text); this.logMessage(`大模型回答: ${result.llmResponse}`, 'success'); // // // 根据文本查找对应视频并切换 // await this.handleTextInput(text); // 并行执行两个操作 // const [result] = await Promise.all([ // this.handleTextInput(text), // chatWithAudioStream(text) // // 视频切换可以立即开始 // ]); // this.logMessage(`大模型回答: ${result.llmResponse}`, 'success'); } catch (error) { this.logMessage(`处理文本失败: ${error.message}`, 'error'); console.error('chatWithAudioStream error:', error); } } } async handleTextInput(text) { // 根据文本查找对应视频 let videoFile = this.videoMapping['default'] || this.defaultVideo; for (const [key, value] of Object.entries(this.videoMapping)) { if (text.toLowerCase().includes(key.toLowerCase())) { videoFile = value; break; } } // 检查当前视频播放状态 const currentVideo = this.recordedVideo; const isVideoPlaying = !currentVideo.paused && !currentVideo.ended && currentVideo.currentTime > 0; if (isVideoPlaying && this.currentVideo !== videoFile) { // 如果当前视频正在播放且需要切换到不同视频 // 可以选择立即切换或等待当前视频播放完成 console.log('当前视频正在播放,准备切换到:', videoFile); // 立即切换(推荐) await this.switchVideoStream(videoFile, 'text', text); // 或者等待当前视频播放完成再切换(可选) // await this.waitForVideoToFinish(); // await this.switchVideoStream(videoFile, 'text', text); } else { // 直接切换 await this.switchVideoStream(videoFile, 'text', text); } // 通知服务器切换视频流 this.socket.emit('switch-video-stream', { videoFile, type: 'text', text }); } // 修改:使用音频处理器的语音录制功能 async startVoiceRecording() { const success = await this.audioProcessor.startRecording(this.localStream); if (success) { this.logMessage('高级语音录制已启动', 'success'); } else { this.logMessage('录音启动失败', 'error'); } } // 修改:停止语音录制 stopVoiceRecording() { this.audioProcessor.stopRecording(); // this.startVoiceButton.disabled = false; // this.stopVoiceButton.disabled = true; // this.startVoiceButton.classList.remove('recording'); // this.voiceStatus.textContent = '点击开始语音输入'; this.logMessage('语音录制已停止', 'info'); } // 处理语音输入结果 async handleVoiceInput(text) { if(text == ""){ console.log("识别到用户未说话") return } // 根据文本查找对应视频 let videoFile = this.videoMapping['default'] || this.defaultVideo; for (const [key, value] of Object.entries(this.videoMapping)) { if (text.toLowerCase().includes(key.toLowerCase())) { videoFile = value; break; } } // 切换到对应的视频流 await this.switchVideoStream(videoFile, 'voice', text); // 通知服务器切换视频流 this.socket.emit('switch-video-stream', { videoFile, type: 'voice', text }); // 调用大模型处理 try { this.logMessage('正在处理语音输入,请稍候...', 'info', this.currentVideoTag); // if (this.currentVideoName === "default"){ const result = await chatWithAudioStream(text); this.logMessage(`大模型回答: ${result.llmResponse}`, 'success'); // } } catch (error) { this.logMessage(`处理语音输入失败: ${error.message}`, 'error'); console.error('chatWithAudioStream error:', error); } } // 删除原有的简单音频处理方法 // processVoiceInput() 和 simulateSpeechRecognition() 方法已被移除 simulateSpeechRecognition() { // 模拟语音识别,随机返回预设的文本 const texts = ['你好', '再见', '谢谢', 'hello', 'goodbye', 'thank you']; return texts[Math.floor(Math.random() * texts.length)]; } returnToDefaultVideo() { this.switchVideoStream(this.defaultVideo, 'default'); this.socket.emit('return-to-default'); this.logMessage(`已返回至默认视频: ${this.defaultVideo}`, 'info'); } updateStatus(message, type) { this.connectionStatus.textContent = message; this.connectionStatus.className = `status ${type}`; // 显示状态元素(仅在连接时显示) if (type === 'connected') { this.connectionStatus.style.display = 'block'; } } updateAudioStatus(message, type) { this.audioStatus.textContent = message; this.audioStatus.className = `status-indicator ${type}`; } logMessage(message, type = 'info') { const logEntry = document.createElement('div'); logEntry.className = type; logEntry.textContent = `[${new Date().toLocaleTimeString()}] ${message}`; this.messageLog.appendChild(logEntry); this.messageLog.scrollTop = this.messageLog.scrollHeight; } checkVideoStreamStatus() { const status = { hasStream: !!this.currentVideoStream, streamTracks: this.currentVideoStream ? this.currentVideoStream.getTracks().length : 0, videoReadyState: this.recordedVideo.readyState, videoPaused: this.recordedVideo.paused, videoCurrentTime: this.recordedVideo.currentTime, videoDuration: this.recordedVideo.duration, currentVideo: this.currentVideo }; this.logMessage(`视频流状态: ${JSON.stringify(status)}`, 'info'); return status; } // 添加图标切换方法 switchToCallingIcon() { const callIcon = document.getElementById('callIcon'); const callingText = document.getElementById('callingText'); const startButton = this.startButton; if (callIcon && callingText && startButton) { callIcon.style.display = 'none'; callingText.style.display = 'block'; startButton.classList.add('calling'); startButton.title = '通话中...'; } } switchToDefaultIcon() { const callIcon = document.getElementById('callIcon'); const callingText = document.getElementById('callingText'); const startButton = this.startButton; if (callIcon && callingText && startButton) { callIcon.style.display = 'block'; callingText.style.display = 'none'; startButton.classList.remove('calling'); startButton.title = '开始通话'; startButton.disabled = false; } } async testAllVideoFiles() { this.logMessage('开始测试所有视频文件...', 'info'); const videoFiles = ['asd.mp4', 'zxc.mp4', 'jkl.mp4']; for (const videoFile of videoFiles) { try { await this.testVideoFile(videoFile); } catch (error) { this.logMessage(`视频文件 ${videoFile} 测试失败: ${error.message}`, 'error'); } } this.logMessage('视频文件测试完成', 'info'); // 检查当前视频流状态 this.checkVideoStreamStatus(); } } // 页面加载完成后初始化应用 document.addEventListener('DOMContentLoaded', () => { console.log('DOMContentLoaded 事件触发'); try { new WebRTCChat(); } catch (error) { console.error('WebRTCChat 初始化失败:', error); } });