From 226ec68525e71f81110b26972c3d8a527e0955fc Mon Sep 17 00:00:00 2001 From: Song367 <601337784@qq.com> Date: Thu, 7 Aug 2025 21:39:08 +0800 Subject: [PATCH] =?UTF-8?q?=E5=BC=83=E7=94=A8webrtc=20=E5=88=87=E6=8D=A2?= =?UTF-8?q?=E8=A7=86=E9=A2=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/index.js | 194 +++++++++++++++++------------------------ src/minimaxi_stream.js | 13 ++- 2 files changed, 90 insertions(+), 117 deletions(-) diff --git a/src/index.js b/src/index.js index 30792d1..0daf054 100644 --- a/src/index.js +++ b/src/index.js @@ -700,16 +700,10 @@ class WebRTCChat { // } // } - // 修改原有的switchVideoStream方法，使用新的平滑切换 + // 修改视频切换方法，直接使用预加载视频切换，不使用WebRTC传输 async switchVideoStream(videoFile, type = '', text = '') { - // 使用平滑切换方法 - return await this.switchVideoStreamSmooth(videoFile, type, text); - } - - // 使用replaceTrack方式切换视频 - async switchVideoWithReplaceTrack(videoFile, type = '', text = '') { try { - this.logMessage(`开始优化切换视频: ${videoFile}`, 'info'); + this.logMessage(`开始切换视频: ${videoFile} (${type})`, 'info'); let newVideoStream; let isUsingPrecreated = false; @@ -730,17 +724,7 @@ class WebRTCChat { } } - const newVideoTrack = newVideoStream.getVideoTracks()[0]; - if (!newVideoTrack) { - throw new Error('新视频流中没有视频轨道'); - } - - // WebRTC轨道替换 - if (this.peerConnection && this.videoSender) { - await this.videoSender.replaceTrack(newVideoTrack); - } - - // 更新本地视频显示 + // 直接切换本地视频显示，不使用WebRTC传输 if (this.recordedVideo) { // 停止当前视频流（但不停止预创建的流） if (this.currentVideoStream && !this.precreatedStreams.has(this.currentVideo)) { @@ -752,17 +736,26 @@ class WebRTCChat { this.currentVideo = videoFile; // 使用预创建流时减少等待时间 - const waitTime = isUsingPrecreated ? 50 : 200; + const waitTime = isUsingPrecreated ? 10 : 50; await new Promise(resolve => setTimeout(resolve, waitTime)); try { await this.recordedVideo.play(); + this.logMessage(`视频切换完成: ${videoFile} (${type})`, 'success'); } catch (playError) { this.logMessage(`视频播放失败: ${playError.message}`, 'error'); } } - this.logMessage(`视频切换完成: ${videoFile} (${type})`, 'success'); + // 更新显示信息 + if (text) { + this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`; + this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success'); + } else { + this.currentVideoName.textContent = `视频流: ${videoFile}`; + this.logMessage(`成功切换到视频流: ${videoFile}`, 'success'); + } + return true; } catch (error) { @@ -805,37 +798,22 @@ class WebRTCChat { return overlay; } - // 新增平滑视频切换方法 + // 新增平滑视频切换方法 - 等待完全传输后切换 async switchVideoStreamSmooth(videoFile, type = '', text = '') { try { - this.logMessage(`开始无闪烁切换视频流: ${videoFile} (${type})`, 'info'); + this.logMessage(`开始等待视频流传输完成: ${videoFile} (${type})`, 'info'); - // 1. 捕获当前视频的最后一帧 + // 1. 获取当前和缓冲视频元素 const currentVideo = this.activeVideoElement === 'main' ? this.recordedVideo : this.recordedVideoBuffer; const bufferVideo = this.activeVideoElement === 'main' ? this.recordedVideoBuffer : this.recordedVideo; - let frameData = null; - if (currentVideo.readyState >= 2 && currentVideo.currentTime > 0) { - frameData = this.captureVideoFrame(currentVideo); - } - - // 2. 创建并显示静态帧覆盖层 - let frameOverlay = null; - if (frameData) { - frameOverlay = this.createFrameOverlay(frameData); - currentVideo.parentElement.appendChild(frameOverlay); - - // 确保覆盖层立即显示 - frameOverlay.offsetHeight; // 强制重绘 - } - - // 3. 检查是否已缓存 + // 2. 检查是否已缓存 const isCached = this.videoStreams.has(videoFile); if (!isCached) { this.showVideoLoading(); } - // 4. 准备新视频流 + // 3. 准备新视频流 let newStream; if (isCached) { const cachedStream = this.videoStreams.get(videoFile); @@ -849,61 +827,76 @@ class WebRTCChat { newStream = await this.createVideoStream(videoFile); } - // 5. 在缓冲视频元素中预加载新视频 + // 4. 在缓冲视频元素中预加载新视频 bufferVideo.srcObject = newStream; - // 6. 增强的视频准备检测 + // 5. 等待视频流完全传输和准备就绪 await new Promise((resolve, reject) => { const timeout = setTimeout(() => { - reject(new Error('视频加载超时')); - }, 8000); // 增加超时时间到10秒 + reject(new Error('视频流传输超时')); + }, 15000); // 增加超时时间到15秒，确保有足够时间传输 let frameCount = 0; - const minFrames = 8; // 增加到8帧确保更稳定的切换 + const minFrames = 10; // 增加最小帧数确保传输完整 + let isStreamReady = false; const onReady = () => { clearTimeout(timeout); bufferVideo.removeEventListener('canplay', onCanPlay); + bufferVideo.removeEventListener('canplaythrough', onCanPlayThrough); bufferVideo.removeEventListener('timeupdate', onTimeUpdate); bufferVideo.removeEventListener('error', onError); + bufferVideo.removeEventListener('loadeddata', onLoadedData); resolve(); }; - const onCanPlay = () => { - // 开始播放新视频 + // 检查视频数据是否完全加载 + const onLoadedData = () => { + this.logMessage(`视频数据开始加载: ${videoFile}`, 'info'); + }; + + // 检查视频是否可以流畅播放（数据传输充足） + const onCanPlayThrough = () => { + this.logMessage(`视频流传输充足，可以流畅播放: ${videoFile}`, 'success'); + isStreamReady = true; + + // 开始播放并等待稳定帧 bufferVideo.play().then(() => { - // 等待多帧渲染确保稳定 - const checkFrames = () => { + const checkFrameStability = () => { if (bufferVideo.currentTime > 0) { frameCount++; - if (frameCount >= minFrames) { - // 额外等待三个渲染周期确保完全稳定 - requestAnimationFrame(() => { - requestAnimationFrame(() => { - requestAnimationFrame(() => { - onReady(); - }); - }); - }); + if (frameCount >= minFrames && isStreamReady) { + // 额外等待确保传输稳定 + setTimeout(() => { + onReady(); + }, 200); // 等待200ms确保传输稳定 } else { - requestAnimationFrame(checkFrames); + requestAnimationFrame(checkFrameStability); } } else { - requestAnimationFrame(checkFrames); + requestAnimationFrame(checkFrameStability); } }; - requestAnimationFrame(checkFrames); + requestAnimationFrame(checkFrameStability); }).catch(reject); }; + // 基本播放准备就绪 + const onCanPlay = () => { + if (!isStreamReady) { + this.logMessage(`视频基本准备就绪，等待完全传输: ${videoFile}`, 'info'); + // 如果还没有canplaythrough事件，继续等待 + } + }; + + // 时间更新监听（备用检查） const onTimeUpdate = () => { - if (bufferVideo.currentTime > 0) { + if (bufferVideo.currentTime > 0 && isStreamReady) { frameCount++; if (frameCount >= minFrames) { - // 额外等待确保帧稳定 setTimeout(() => { onReady(); - }, 100); // 增加等待时间到100ms + }, 200); } } }; @@ -911,72 +904,63 @@ class WebRTCChat { const onError = (error) => { clearTimeout(timeout); bufferVideo.removeEventListener('canplay', onCanPlay); + bufferVideo.removeEventListener('canplaythrough', onCanPlayThrough); bufferVideo.removeEventListener('timeupdate', onTimeUpdate); bufferVideo.removeEventListener('error', onError); + bufferVideo.removeEventListener('loadeddata', onLoadedData); reject(error); }; - if (bufferVideo.readyState >= 3) { - onCanPlay(); + // 检查当前状态 + if (bufferVideo.readyState >= 4) { // HAVE_ENOUGH_DATA + onCanPlayThrough(); } else { + bufferVideo.addEventListener('loadeddata', onLoadedData); bufferVideo.addEventListener('canplay', onCanPlay); + bufferVideo.addEventListener('canplaythrough', onCanPlayThrough); bufferVideo.addEventListener('timeupdate', onTimeUpdate); bufferVideo.addEventListener('error', onError); } }); - // 7. 三重确认新视频已准备好 + // 6. 最终确认视频流完全准备就绪 await new Promise(resolve => { let confirmCount = 0; - const maxConfirms = 5; // 增加确认次数 + const maxConfirms = 3; const finalCheck = () => { - if (bufferVideo.readyState >= 2 && bufferVideo.currentTime > 0) { + if (bufferVideo.readyState >= 4 && bufferVideo.currentTime > 0) { confirmCount++; if (confirmCount >= maxConfirms) { - // 再等待三个渲染周期确保完全准备好 - requestAnimationFrame(() => { - requestAnimationFrame(() => { - requestAnimationFrame(() => { - resolve(); - }); - }); - }); + this.logMessage(`视频流传输完成，准备切换: ${videoFile}`, 'success'); + resolve(); } else { - requestAnimationFrame(finalCheck); + setTimeout(finalCheck, 50); // 每50ms检查一次 } } else { - requestAnimationFrame(finalCheck); + setTimeout(finalCheck, 50); } }; finalCheck(); }); - // 8. 立即切换视频显示（无过渡） + // 7. 执行无缝切换（当前视频继续播放直到新视频完全准备好） bufferVideo.style.zIndex = '2'; currentVideo.style.zIndex = '1'; - // 9. 延迟移除静态帧覆盖层（进一步增加显示时长） - if (frameOverlay) { - // 等待更长时间确保新视频完全显示 - setTimeout(() => { - if (frameOverlay && frameOverlay.parentElement) { - frameOverlay.remove(); - } - }, 300); // 增加到300ms - } + this.logMessage(`视频切换完成: ${videoFile}`, 'success'); - // 10. 隐藏加载指示器 + // 8. 隐藏加载指示器 if (!isCached) { this.hideVideoLoading(); } - // 11. 更新状态 + // 9. 更新状态 this.currentVideoStream = newStream; this.currentVideo = videoFile; this.activeVideoElement = this.activeVideoElement === 'main' ? 'buffer' : 'main'; - // 12. 延迟清理旧视频流 + // 10. 延迟清理旧视频流 setTimeout(() => { if (currentVideo.srcObject && currentVideo.srcObject !== newStream) { currentVideo.srcObject.getTracks().forEach(track => track.stop()); @@ -984,7 +968,7 @@ class WebRTCChat { } }, 1000); - // 13. 更新WebRTC连接 + // 11. 更新WebRTC连接 if (this.peerConnection && this.videoSender) { const newVideoTrack = newStream.getVideoTracks()[0]; if (newVideoTrack) { @@ -992,7 +976,7 @@ class WebRTCChat { } } - // 14. 更新显示信息 + // 12. 更新显示信息 if (text) { this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`; this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success'); @@ -1004,12 +988,6 @@ class WebRTCChat { } catch (error) { this.logMessage(`视频流切换失败: ${error.message}`, 'error'); - // 清理可能残留的覆盖层 - const overlay = document.getElementById('video-frame-overlay'); - if (overlay) { - overlay.remove(); - } - this.hideVideoLoading(); // 回退到默认视频 @@ -1283,18 +1261,8 @@ class WebRTCChat { this.peerConnection.addTrack(track, this.localStream); }); - // 添加初始视频流（默认视频） - try { - const initialVideoStream = await this.createVideoStream(this.defaultVideo); - const videoTrack = initialVideoStream.getVideoTracks()[0]; - if (videoTrack) { - this.videoSender = this.peerConnection.addTrack(videoTrack, initialVideoStream); - this.currentVideoStream = initialVideoStream; - this.logMessage('初始视频轨道已添加到WebRTC连接', 'success'); - } - } catch (error) { - this.logMessage(`添加初始视频轨道失败: ${error.message}`, 'error'); - } + // 移除视频轨道添加逻辑，只使用音频进行WebRTC通信 + // 视频将直接在本地切换，不通过WebRTC传输 // 处理远程流 this.peerConnection.ontrack = (event) => { diff --git a/src/minimaxi_stream.js b/src/minimaxi_stream.js index ecc757b..3b8a5ce 100644 --- a/src/minimaxi_stream.js +++ b/src/minimaxi_stream.js @@ -68,10 +68,10 @@ async function processAudioQueue() { const sayName = '8-4-sh' const targetVideo = window.webrtcApp.interactionVideo // 如果是第一个音频片段，触发视频切换 - if (isFirstChunk && sayName != window.webrtcApp.currentVideoTag && window.webrtcApp && window.webrtcApp.switchVideoWithReplaceTrack) { + if (isFirstChunk && sayName != window.webrtcApp.currentVideoTag && window.webrtcApp && window.webrtcApp.switchVideoStream) { try { console.log('--------------触发视频切换:', sayName); - window.webrtcApp.switchVideoWithReplaceTrack(targetVideo, 'audio', '8-4-sh'); + window.webrtcApp.switchVideoStream(targetVideo, 'audio', '8-4-sh'); isFirstChunk = false; window.webrtcApp.currentVideoTag = sayName; } catch (error) { @@ -86,13 +86,18 @@ async function processAudioQueue() { } isProcessingQueue = false; + + // 等待当前音频播放完成后再切换回默认视频 + while (isPlaying) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + const text = 'default' - // await new Promise(resolve => setTimeout(resolve, 500)); console.log("音频结束------------------------：", window.webrtcApp.currentVideoTag, isPlaying) if (window.webrtcApp.currentVideoTag != text) { isFirstChunk = true window.webrtcApp.currentVideoTag = text - window.webrtcApp.switchVideoWithReplaceTrack(window.webrtcApp.defaultVideo, 'audio', text); + window.webrtcApp.switchVideoStream(window.webrtcApp.defaultVideo, 'audio', text); } console.log('音频队列处理完成'); }