9a1bd0acfd
All checks were successful
Gitea Actions Demo / Explore-Gitea-Actions (push) Successful in 2m53s
1711 lines
65 KiB
JavaScript
1711 lines
65 KiB
JavaScript
console.log('视频文件:');
|
||
// WebRTC 音视频通话应用
|
||
// import { chatWithAudioStream } from './chat_with_audio.js';
|
||
import { chatWithAudioStream, initializeHistoryMessage } from './chat_with_audio.js';
|
||
import { AudioProcessor } from './audio_processor.js';
|
||
|
||
// 在应用初始化时调用
|
||
class WebRTCChat {
|
||
constructor() {
|
||
console.log('WebRTCChat 构造函数开始执行');
|
||
|
||
// 初始化历史消息（异步）
|
||
this.initializeHistory();
|
||
|
||
this.socket = null;
|
||
this.localStream = null;
|
||
this.peerConnection = null;
|
||
this.isRecording = false;
|
||
this.mediaRecorder = null;
|
||
this.audioChunks = [];
|
||
this.videoMapping = {};
|
||
this.defaultVideo = 'chang.mp4';
|
||
this.interactionVideo = 'chang.mp4';
|
||
this.currentVideoTag = 'default';
|
||
this.currentVideo = null;
|
||
this.videoStreams = new Map(); // 存储不同视频的MediaStream
|
||
this.currentVideoStream = null;
|
||
this.precreatedStreams = new Map(); // 预创建的视频流
|
||
this.importantVideos = ['d-3s.mp4', 's-1.mp4']; // 重要视频列表
|
||
this.isInitialized = false;
|
||
// 添加视频加载状态标志
|
||
this.isVideoReady = false;
|
||
this.isDefaultVideoLoaded = false;
|
||
this.retryCount = 0; // 添加重试计数器
|
||
|
||
// 添加视频相关属性
|
||
this.videoSender = null; // WebRTC视频发送器
|
||
this.currentVideoStream = null; // 当前视频流
|
||
|
||
// 初始化音频处理器
|
||
console.log('开始初始化音频处理器');
|
||
// 初始化音频处理器
|
||
this.audioProcessor = new AudioProcessor({
|
||
onSpeechStart: () => {
|
||
// this.voiceStatus.textContent = '检测到语音，开始录音...';
|
||
this.logMessage('检测到语音，开始录音...', 'info');
|
||
},
|
||
onSpeechEnd: () => {
|
||
// 语音结束回调
|
||
},
|
||
onRecognitionResult: (text) => {
|
||
// ASRTEXT = text;
|
||
// this.voiceStatus.textContent = '识别完成';
|
||
this.logMessage(`语音识别结果: ${text}`, 'success');
|
||
this.handleVoiceInput(text);
|
||
},
|
||
onError: (error) => {
|
||
// this.voiceStatus.textContent = '识别失败';
|
||
this.logMessage(error, 'error');
|
||
},
|
||
onStatusUpdate: (message, status) => {
|
||
// this.voiceStatus.textContent = message;
|
||
}
|
||
});
|
||
|
||
console.log('WebRTC 聊天应用初始化完成');
|
||
|
||
this.initializeSocket();
|
||
this.initializeElements();
|
||
this.loadVideoMapping();
|
||
this.loadVideoList();
|
||
|
||
// 只预加载视频资源，不显示视频
|
||
this.preloadVideoResources();
|
||
this.bindEvents();
|
||
|
||
// 在初始化完成后预加载常用视频
|
||
// setTimeout(() => {
|
||
// this.logMessage('开始预加载常用视频...', 'info');
|
||
// this.preloadCommonVideos().catch(error => {
|
||
// this.logMessage(`预加载过程出错: ${error.message}`, 'error');
|
||
// });
|
||
// }, 500);
|
||
|
||
// 预创建重要视频流
|
||
setTimeout(() => {
|
||
this.precreateImportantVideos().catch(error => {
|
||
this.logMessage(`预创建重要视频流失败: ${error.message}`, 'error');
|
||
});
|
||
}, 1000);
|
||
|
||
window.webrtcApp = this;
|
||
}
|
||
|
||
// 新增方法：预加载视频资源
|
||
async preloadVideoResources() {
|
||
try {
|
||
await this.loadDefaultVideo();
|
||
// 预创建视频流但不设置到video元素
|
||
const defaultStream = await this.createVideoStreamOptimized(this.defaultVideo);
|
||
this.precreatedStreams.set(this.defaultVideo, defaultStream);
|
||
|
||
this.isVideoReady = true;
|
||
this.isDefaultVideoLoaded = true;
|
||
|
||
// 启用开始通话按钮
|
||
// if (this.startButton) {
|
||
// this.startButton.disabled = false;
|
||
// this.startButton.style.opacity = '1';
|
||
// }
|
||
|
||
console.log('视频资源预加载完成，可以开始通话', 'success');
|
||
} catch (error) {
|
||
this.logMessage(`视频资源预加载失败: ${error.message}`, 'error');
|
||
this.isVideoReady = false;
|
||
}
|
||
}
|
||
|
||
initializeElements() {
|
||
// 视频元素
|
||
this.localVideo = document.getElementById('localVideo');
|
||
this.remoteVideo = document.getElementById('remoteVideo');
|
||
this.recordedVideo = document.getElementById('recordedVideo');
|
||
this.recordedVideoBuffer = document.getElementById('recordedVideoBuffer'); // 新增缓冲视频元素
|
||
this.videoLoading = document.getElementById('videoLoading'); // 加载指示器
|
||
|
||
// 头像和视频容器元素
|
||
this.avatarContainer = document.getElementById('avatarContainer');
|
||
this.videoContainer = document.getElementById('videoContainer');
|
||
|
||
// 当前活跃的视频元素标识
|
||
this.activeVideoElement = 'main'; // 'main' 或 'buffer'
|
||
|
||
// 音频状态元素
|
||
this.audioStatus = document.getElementById('audioStatus');
|
||
|
||
// 按钮元素
|
||
this.startButton = document.getElementById('startButton');
|
||
this.stopButton = document.getElementById('stopButton');
|
||
this.muteButton = document.getElementById('muteButton');
|
||
this.sendTextButton = document.getElementById('sendTextButton');
|
||
|
||
// 初始状态下禁用开始通话按钮，直到视频加载完成
|
||
if (this.startButton) {
|
||
this.startButton.disabled = true;
|
||
this.startButton.style.opacity = '0.5';
|
||
// this.startButton.title = '加载中，请稍候...';
|
||
}
|
||
|
||
// this.startVoiceButton = document.getElementById('startVoiceButton');
|
||
// this.stopVoiceButton = document.getElementById('stopVoiceButton');
|
||
// this.defaultVideoButton = document.getElementById('defaultVideoButton');
|
||
// this.testVideoButton = document.getElementById('testVideoButton'); // 新增测试按钮
|
||
|
||
// 输入元素
|
||
this.textInput = document.getElementById('textInput');
|
||
// this.voiceStatus = document.getElementById('voiceStatus');
|
||
|
||
// 状态元素
|
||
this.connectionStatus = document.getElementById('connectionStatus');
|
||
this.messageLog = document.getElementById('messageLog');
|
||
this.currentVideoName = document.getElementById('currentVideoName');
|
||
this.videoList = document.getElementById('videoList');
|
||
|
||
// 等待连接提示元素
|
||
this.connectionWaiting = document.getElementById('connectionWaiting');
|
||
}
|
||
|
||
initializeSocket() {
|
||
this.socket = io();
|
||
|
||
this.socket.on('connect', () => {
|
||
this.updateStatus('已连接到服务器', 'connected');
|
||
this.logMessage('已连接到服务器', 'success');
|
||
});
|
||
|
||
this.socket.on('disconnect', () => {
|
||
this.connectionStatus.style.display = 'none';
|
||
this.logMessage('与服务器断开连接', 'error');
|
||
});
|
||
|
||
// WebRTC 信令处理
|
||
this.socket.on('offer', (data) => {
|
||
this.handleOffer(data);
|
||
});
|
||
|
||
this.socket.on('answer', (data) => {
|
||
this.handleAnswer(data);
|
||
});
|
||
|
||
this.socket.on('ice-candidate', (data) => {
|
||
this.handleIceCandidate(data);
|
||
});
|
||
|
||
// 视频流切换处理
|
||
this.socket.on('video-stream-switched', (data) => {
|
||
this.logMessage(`收到视频流切换指令: ${data.videoFile} (${data.type}) 来自用户: ${data.from}`, 'info');
|
||
this.switchVideoStream(data.videoFile, data.type, data.text);
|
||
});
|
||
|
||
// 通话开始处理
|
||
this.socket.on('call-started', (data) => {
|
||
console.log('通话已开始', 'success');
|
||
// 移除这里的视频流启动，因为现在在startCall中处理
|
||
});
|
||
|
||
// 场景切换处理
|
||
this.socket.on('scene-switched', (data) => {
|
||
this.logMessage(`场景已切换到: ${data.scene.name}`, 'info');
|
||
|
||
// 更新视频映射
|
||
this.videoMapping = data.mapping;
|
||
this.interactionVideo = data.mapping.interactionVideo;
|
||
this.defaultVideo = data.mapping.defaultVideo;
|
||
|
||
// 立即切换到新场景的默认视频
|
||
this.switchVideoStream(this.defaultVideo, 'scene-change');
|
||
|
||
console.log('场景切换完成，新的视频配置:', {
|
||
defaultVideo: this.defaultVideo,
|
||
interactionVideo: this.interactionVideo,
|
||
tag: data.mapping.tag
|
||
});
|
||
});
|
||
}
|
||
|
||
async initializeHistory() {
|
||
try {
|
||
await initializeHistoryMessage(100);
|
||
|
||
console.log('历史消息初始化完成');
|
||
} catch (error) {
|
||
console.error('历史消息初始化失败:', error);
|
||
}
|
||
}
|
||
|
||
async loadVideoMapping() {
|
||
try {
|
||
const response = await fetch('/api/video-mapping');
|
||
const data = await response.json();
|
||
this.videoMapping = data.mapping;
|
||
this.interactionVideo = data.mapping['8-4-sh'];
|
||
this.defaultVideo = data.mapping["default"];
|
||
console.log('映射加载成功', 'success');
|
||
} catch (error) {
|
||
this.logMessage('加载视频映射失败: ' + error.message, 'error');
|
||
}
|
||
}
|
||
|
||
async loadDefaultVideo() {
|
||
try {
|
||
const response = await fetch('/api/default-video');
|
||
const data = await response.json();
|
||
this.defaultVideo = data.defaultVideo;
|
||
this.logMessage('默认视频配置加载成功', 'success');
|
||
} catch (error) {
|
||
this.logMessage('加载默认视频配置失败: ' + error.message, 'error');
|
||
}
|
||
}
|
||
|
||
async loadVideoList() {
|
||
try {
|
||
const response = await fetch('/api/videos');
|
||
const data = await response.json();
|
||
this.renderVideoList(data.videos);
|
||
this.logMessage('视频列表加载成功', 'success');
|
||
} catch (error) {
|
||
this.logMessage('加载视频列表失败: ' + error.message, 'error');
|
||
}
|
||
}
|
||
|
||
renderVideoList(videos) {
|
||
this.videoList.innerHTML = '';
|
||
videos.forEach(video => {
|
||
const videoItem = document.createElement('div');
|
||
videoItem.className = 'video-item';
|
||
videoItem.textContent = video;
|
||
videoItem.onclick = () => this.selectVideo(video);
|
||
this.videoList.appendChild(videoItem);
|
||
});
|
||
}
|
||
|
||
selectVideo(videoFile) {
|
||
// 移除之前的active类
|
||
document.querySelectorAll('.video-item').forEach(item => {
|
||
item.classList.remove('active');
|
||
});
|
||
|
||
// 添加active类到选中的视频
|
||
event.target.classList.add('active');
|
||
|
||
// 切换到选中的视频流
|
||
this.switchVideoStream(videoFile, 'manual');
|
||
|
||
// 通知服务器切换视频流
|
||
this.socket.emit('switch-video-stream', {
|
||
videoFile,
|
||
type: 'manual'
|
||
});
|
||
}
|
||
|
||
async startDefaultVideoStream() {
|
||
try {
|
||
console.log('开始创建默认视频流', 'info');
|
||
|
||
// 显示视频元素
|
||
if (this.recordedVideo) {
|
||
this.recordedVideo.style.display = 'block';
|
||
}
|
||
|
||
// 添加加载状态
|
||
this.recordedVideo.classList.add('loading');
|
||
|
||
// 创建默认视频的MediaStream
|
||
let defaultStream = this.precreatedStreams.get(this.defaultVideo);
|
||
|
||
// 检查流是否有效，如果无效则重新创建
|
||
if (!defaultStream || defaultStream.getTracks().length === 0 ||
|
||
defaultStream.getTracks().some(track => track.readyState === 'ended')) {
|
||
console.log('预创建流无效，重新创建默认视频流');
|
||
try {
|
||
defaultStream = await this.createVideoStreamOptimized(this.defaultVideo);
|
||
this.precreatedStreams.set(this.defaultVideo, defaultStream);
|
||
} catch (createError) {
|
||
throw new Error(`重新创建默认视频流失败: ${createError.message}`);
|
||
}
|
||
}
|
||
|
||
// 等待流稳定
|
||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||
|
||
// 再次检查流是否有效
|
||
if (!defaultStream || defaultStream.getTracks().length === 0) {
|
||
throw new Error('默认视频流创建失败');
|
||
}
|
||
|
||
// 设置视频流
|
||
this.recordedVideoBuffer.style.zIndex = "1";
|
||
this.recordedVideo.style.zIndex = "2";
|
||
this.currentVideoStream = defaultStream;
|
||
this.recordedVideo.srcObject = defaultStream;
|
||
this.recordedVideoBuffer.srcObject = this.precreatedStreams.get(this.interactionVideo);
|
||
// this.recordedVideoBuffer.style.zIndex = "10"
|
||
// this.recordedVideoBuffer.style.opacity = "1"; // 添加这行
|
||
// this.recordedVideo.style.zIndex = "-1"
|
||
// this.recordedVideo.style.opacity = "0";
|
||
|
||
// this.recordedVideoBuffer.play();
|
||
|
||
|
||
|
||
this.currentVideo = this.defaultVideo;
|
||
this.currentVideoName.textContent = `默认视频: ${this.defaultVideo}`;
|
||
|
||
// 等待视频元素准备就绪
|
||
await new Promise(resolve => {
|
||
const checkReady = () => {
|
||
if (this.recordedVideo.readyState >= 2) { // HAVE_CURRENT_DATA
|
||
resolve();
|
||
} else {
|
||
setTimeout(checkReady, 100);
|
||
}
|
||
};
|
||
checkReady();
|
||
});
|
||
this.avatarContainer.style.display = 'none';
|
||
|
||
// 隐藏等待连接提示
|
||
this.hideConnectionWaiting();
|
||
|
||
// 确保视频开始播放
|
||
try {
|
||
await this.recordedVideo.play();
|
||
// this.logMessage('默认视频开始播放', 'success');
|
||
|
||
// 移除加载状态，添加播放状态
|
||
this.recordedVideo.classList.remove('loading');
|
||
this.recordedVideo.classList.add('playing');
|
||
} catch (playError) {
|
||
// this.logMessage(`默认视频播放失败: ${playError.message}`, 'error');
|
||
this.recordedVideo.classList.remove('loading');
|
||
}
|
||
|
||
// 再次检查视频是否准备就绪
|
||
// this.startButton.disabled = false;
|
||
|
||
console.log('默认流创建成功', 'success');
|
||
} catch (error) {
|
||
console.log('创建默认视频流失败: ' + error.message, 'error');
|
||
|
||
// 隐藏视频元素
|
||
if (this.recordedVideo) {
|
||
this.recordedVideo.style.display = 'none';
|
||
}
|
||
this.recordedVideo.classList.remove('loading');
|
||
|
||
// 添加重试机制
|
||
if (!this.retryCount) this.retryCount = 0;
|
||
if (this.retryCount < 2) {
|
||
this.retryCount++;
|
||
console.log(`尝试重新创建默认视频流 (${this.retryCount}/2)`);
|
||
setTimeout(() => this.startDefaultVideoStream(), 1000);
|
||
} else {
|
||
this.retryCount = 0;
|
||
console.log('默认视频流创建失败，已达到最大重试次数', 'error');
|
||
}
|
||
}
|
||
}
|
||
|
||
async testVideoFile(videoFile) {
|
||
return new Promise((resolve, reject) => {
|
||
const testVideo = document.createElement('video');
|
||
testVideo.src = `/videos/${videoFile}`;
|
||
testVideo.muted = true;
|
||
|
||
testVideo.onloadedmetadata = () => {
|
||
this.logMessage(`视频文件测试成功: ${videoFile} (${testVideo.videoWidth}x${testVideo.videoHeight})`, 'success');
|
||
resolve(true);
|
||
};
|
||
|
||
testVideo.onerror = () => {
|
||
this.logMessage(`视频文件测试失败: ${videoFile}`, 'error');
|
||
reject(new Error(`视频文件不存在或无法加载: ${videoFile}`));
|
||
};
|
||
|
||
// 设置超时
|
||
setTimeout(() => {
|
||
reject(new Error(`视频文件加载超时: ${videoFile}`));
|
||
}, 10000);
|
||
});
|
||
}
|
||
|
||
// 预创建重要视频流
|
||
async precreateImportantVideos() {
|
||
if (this.isInitialized) return;
|
||
|
||
console.log('开始预创建重要流...', 'info');
|
||
|
||
for (const videoFile of [this.interactionVideo, this.defaultVideo]) {
|
||
try {
|
||
const stream = await this.createVideoStreamOptimized(videoFile);
|
||
this.precreatedStreams.set(videoFile, stream);
|
||
this.logMessage(`预创建视频流成功: ${videoFile}`, 'success');
|
||
} catch (error) {
|
||
this.logMessage(`预创建视频流失败: ${videoFile} - ${error.message}`, 'error');
|
||
}
|
||
}
|
||
// 启用开始通话按钮
|
||
if (this.startButton) {
|
||
this.startButton.disabled = false;
|
||
this.startButton.style.opacity = '1';
|
||
}
|
||
|
||
this.isInitialized = true;
|
||
|
||
this.logMessage('重要视频流预创建完成', 'success');
|
||
}
|
||
|
||
// 优化的视频流创建方法
|
||
async createVideoStreamOptimized(videoFile) {
|
||
try {
|
||
// 先测试视频文件是否存在
|
||
await this.testVideoFile(videoFile);
|
||
|
||
// 创建video元素
|
||
const video = document.createElement('video');
|
||
video.src = `/videos/${videoFile}`;
|
||
video.muted = true;
|
||
video.loop = true;
|
||
video.autoplay = false;
|
||
video.crossOrigin = 'anonymous';
|
||
video.playsInline = true;
|
||
video.preload = 'auto';
|
||
|
||
// 等待视频加载
|
||
await new Promise((resolve, reject) => {
|
||
video.onloadeddata = () => {
|
||
video.currentTime = 0;
|
||
resolve();
|
||
};
|
||
video.onerror = reject;
|
||
});
|
||
|
||
// 创建优化的Canvas
|
||
const canvas = document.createElement('canvas');
|
||
const ctx = canvas.getContext('2d', {
|
||
alpha: false, // 禁用透明度以提高性能
|
||
willReadFrequently: false // 优化Canvas性能
|
||
});
|
||
|
||
canvas.width = video.videoWidth || 640;
|
||
canvas.height = video.videoHeight || 480;
|
||
|
||
// 绘制第一帧
|
||
await new Promise((resolve) => {
|
||
const drawFirstFrame = () => {
|
||
if (video.readyState >= video.HAVE_CURRENT_DATA) {
|
||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||
resolve();
|
||
} else {
|
||
setTimeout(drawFirstFrame, 10);
|
||
}
|
||
};
|
||
drawFirstFrame();
|
||
});
|
||
|
||
// 启动视频播放
|
||
await video.play();
|
||
|
||
// 优化的帧绘制 - 使用更高效的节流
|
||
let animationId;
|
||
let lastFrameTime = 0;
|
||
const targetFPS = 30;
|
||
const frameInterval = 1000 / targetFPS;
|
||
|
||
const drawFrame = (currentTime) => {
|
||
if (currentTime - lastFrameTime >= frameInterval) {
|
||
if (video.readyState >= video.HAVE_CURRENT_DATA) {
|
||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||
}
|
||
lastFrameTime = currentTime;
|
||
}
|
||
animationId = requestAnimationFrame(drawFrame);
|
||
};
|
||
|
||
animationId = requestAnimationFrame(drawFrame);
|
||
|
||
// 创建MediaStream
|
||
const stream = canvas.captureStream(30);
|
||
|
||
// 存储清理函数
|
||
stream._cleanup = () => {
|
||
if (animationId) {
|
||
cancelAnimationFrame(animationId);
|
||
}
|
||
video.pause();
|
||
video.src = '';
|
||
video.load();
|
||
};
|
||
|
||
return stream;
|
||
|
||
} catch (error) {
|
||
this.logMessage(`优化视频流创建失败 ${videoFile}: ${error.message}`, 'error');
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
async createVideoStream(videoFile) {
|
||
// 检查缓存，但为每个视频创建独立的播放实例
|
||
const cacheKey = `${videoFile}_${Date.now()}`; // 添加时间戳确保唯一性
|
||
|
||
try {
|
||
this.logMessage(`开始创建视频流: ${videoFile}`, 'info');
|
||
|
||
// 先测试视频文件是否存在
|
||
await this.testVideoFile(videoFile);
|
||
|
||
// 创建video元素来加载视频
|
||
const video = document.createElement('video');
|
||
video.src = `/videos/${videoFile}`;
|
||
video.muted = true;
|
||
video.loop = true;
|
||
video.autoplay = false; // 手动控制播放
|
||
video.crossOrigin = 'anonymous';
|
||
video.playsInline = true;
|
||
|
||
// 预加载视频但不播放
|
||
video.preload = 'auto';
|
||
|
||
// 等待视频加载完成
|
||
await new Promise((resolve, reject) => {
|
||
video.onloadeddata = () => {
|
||
this.logMessage(`视频数据加载完成: ${videoFile}`, 'info');
|
||
// 确保从第一帧开始
|
||
video.currentTime = 0;
|
||
resolve();
|
||
};
|
||
video.onerror = (error) => {
|
||
this.logMessage(`视频加载失败: ${videoFile}`, 'error');
|
||
reject(error);
|
||
};
|
||
});
|
||
|
||
// 创建MediaStream
|
||
const canvas = document.createElement('canvas');
|
||
const ctx = canvas.getContext('2d');
|
||
|
||
// 设置canvas尺寸为视频尺寸
|
||
canvas.width = video.videoWidth || 640;
|
||
canvas.height = video.videoHeight || 480;
|
||
|
||
this.logMessage(`Canvas尺寸: ${canvas.width}x${canvas.height}`, 'info');
|
||
|
||
// 确保第一帧立即绘制，避免黑屏
|
||
await new Promise((resolve) => {
|
||
const drawFirstFrame = () => {
|
||
if (video.readyState >= video.HAVE_CURRENT_DATA) {
|
||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||
this.logMessage('已绘制第一帧到Canvas', 'info');
|
||
resolve();
|
||
} else {
|
||
setTimeout(drawFirstFrame, 10);
|
||
}
|
||
};
|
||
drawFirstFrame();
|
||
});
|
||
|
||
// 开始播放视频
|
||
try {
|
||
await video.play();
|
||
this.logMessage(`视频开始播放: ${videoFile}`, 'info');
|
||
} catch (playError) {
|
||
this.logMessage(`视频播放失败: ${playError.message}`, 'error');
|
||
}
|
||
|
||
// 等待视频真正开始播放
|
||
await new Promise(resolve => {
|
||
const checkPlay = () => {
|
||
if (video.readyState >= video.HAVE_CURRENT_DATA && !video.paused) {
|
||
resolve();
|
||
} else {
|
||
setTimeout(checkPlay, 50);
|
||
}
|
||
};
|
||
checkPlay();
|
||
});
|
||
|
||
// 绘制视频到canvas
|
||
let lastDrawTime = 0;
|
||
let isDrawing = false;
|
||
const drawFrame = () => {
|
||
const now = performance.now();
|
||
if (video.readyState >= video.HAVE_CURRENT_DATA && !isDrawing && (now - lastDrawTime > 16)) {
|
||
isDrawing = true;
|
||
lastDrawTime = now;
|
||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||
isDrawing = false;
|
||
}
|
||
requestAnimationFrame(drawFrame);
|
||
};
|
||
|
||
// 开始绘制帧
|
||
drawFrame();
|
||
|
||
// 从canvas创建MediaStream
|
||
const stream = canvas.captureStream(30);
|
||
|
||
// 等待流稳定
|
||
await new Promise(resolve => {
|
||
setTimeout(resolve, 200); // 减少等待时间
|
||
});
|
||
|
||
this.logMessage(`视频流创建成功: ${videoFile}`, 'success');
|
||
|
||
// 使用有限缓存策略（最多缓存3个视频流）
|
||
if (this.videoStreams.size >= 3) {
|
||
const firstKey = this.videoStreams.keys().next().value;
|
||
const oldStream = this.videoStreams.get(firstKey);
|
||
if (oldStream) {
|
||
oldStream.getTracks().forEach(track => track.stop());
|
||
}
|
||
this.videoStreams.delete(firstKey);
|
||
}
|
||
|
||
this.videoStreams.set(videoFile, stream);
|
||
|
||
return stream;
|
||
} catch (error) {
|
||
this.logMessage(`创建视频流失败 ${videoFile}: ${error.message}`, 'error');
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
// 在应用初始化时预加载常用视频
|
||
async preloadCommonVideos() {
|
||
// 获取所有可能需要的视频
|
||
console.log("default video, interaction video", [this.defaultVideo, this.interactionVideo])
|
||
const videosToPreload = new Set([this.defaultVideo, this.interactionVideo]);
|
||
|
||
// 添加视频映射中的所有视频
|
||
// Object.values(this.videoMapping).forEach(video => {
|
||
// videosToPreload.add(video);
|
||
// });
|
||
|
||
// 特别确保添加了5.mp4（从日志看这是常用视频）
|
||
// videosToPreload.add('d-0.mp4');
|
||
|
||
// 并行预加载，提高效率
|
||
const preloadPromises = Array.from(videosToPreload).map(async (videoFile) => {
|
||
try {
|
||
this.logMessage(`预加载视频: ${videoFile}`, 'info');
|
||
await this.createVideoStream(videoFile);
|
||
this.logMessage(`预加载完成: ${videoFile}`, 'success');
|
||
} catch (error) {
|
||
this.logMessage(`预加载失败: ${videoFile}: ${error.message}`, 'error');
|
||
}
|
||
});
|
||
|
||
await Promise.allSettled(preloadPromises);
|
||
}
|
||
|
||
// async switchVideoStream(videoFile, type = '', text = '') {
|
||
// try {
|
||
// this.logMessage(`开始切换视频流: ${videoFile} (${type})`, 'info');
|
||
|
||
// // 检查是否已缓存
|
||
// const isCached = this.videoStreams.has(videoFile);
|
||
|
||
// // 如果已缓存，直接使用，避免loading状态
|
||
// if (isCached) {
|
||
// const cachedStream = this.videoStreams.get(videoFile);
|
||
// if (cachedStream && cachedStream.getTracks().length > 0) {
|
||
// // 直接切换到缓存的流
|
||
// this.currentVideoStream = cachedStream;
|
||
// this.recordedVideo.srcObject = cachedStream;
|
||
// this.currentVideo = videoFile;
|
||
|
||
// // 立即播放，无需loading状态
|
||
// await this.recordedVideo.play();
|
||
// this.recordedVideo.classList.add('playing');
|
||
|
||
// this.logMessage(`使用缓存视频流: ${videoFile}`, 'success');
|
||
// return;
|
||
// }
|
||
// }
|
||
|
||
// // 未缓存的视频才显示loading状态
|
||
// this.recordedVideo.classList.add('loading');
|
||
|
||
// // 先创建新的视频流
|
||
// const newStream = await this.createVideoStream(videoFile);
|
||
|
||
// // 减少等待时间
|
||
// await new Promise(resolve => setTimeout(resolve, 100));
|
||
|
||
// // 检查流是否有效
|
||
// if (!newStream || newStream.getTracks().length === 0) {
|
||
// throw new Error('创建的视频流无效');
|
||
// }
|
||
|
||
// // 设置新的视频流
|
||
// this.currentVideoStream = newStream;
|
||
// this.recordedVideo.srcObject = newStream;
|
||
// this.currentVideo = videoFile;
|
||
|
||
// // 确保视频开始播放
|
||
// try {
|
||
// await this.recordedVideo.play();
|
||
// this.logMessage('视频元素开始播放', 'info');
|
||
|
||
// // 移除加载状态，添加播放状态
|
||
// this.recordedVideo.classList.remove('loading');
|
||
// this.recordedVideo.classList.add('playing');
|
||
// } catch (playError) {
|
||
// this.logMessage(`视频播放失败: ${playError.message}`, 'error');
|
||
// this.recordedVideo.classList.remove('loading');
|
||
// }
|
||
|
||
// // 现在停止旧的视频流
|
||
// if (this.currentVideoStream !== newStream) {
|
||
// const oldStream = this.currentVideoStream;
|
||
// setTimeout(() => {
|
||
// if (oldStream) {
|
||
// oldStream.getTracks().forEach(track => {
|
||
// track.stop();
|
||
// this.logMessage(`已停止旧轨道: ${track.kind}`, 'info');
|
||
// });
|
||
// }
|
||
// }, 1000); // 延迟1秒停止旧流，确保新流已经稳定
|
||
// }
|
||
|
||
// if (text) {
|
||
// this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`;
|
||
// this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success');
|
||
// } else {
|
||
// this.currentVideoName.textContent = `视频流: ${videoFile}`;
|
||
// this.logMessage(`成功切换到视频流: ${videoFile}`, 'success');
|
||
// }
|
||
|
||
// // 检查切换后的状态
|
||
// setTimeout(() => {
|
||
// this.checkVideoStreamStatus();
|
||
// }, 1000);
|
||
|
||
// } catch (error) {
|
||
// this.logMessage(`切换视频流失败: ${error.message}`, 'error');
|
||
// this.recordedVideo.classList.remove('loading');
|
||
|
||
// // 如果切换失败，尝试回到默认视频
|
||
// if (videoFile !== this.defaultVideo) {
|
||
// this.logMessage('尝试回到默认视频', 'info');
|
||
// await this.switchVideoStream(this.defaultVideo, 'fallback');
|
||
// }
|
||
// }
|
||
// }
|
||
|
||
// 修改视频切换方法，直接使用预加载视频切换，不使用WebRTC传输
|
||
async switchVideoStream(videoFile, type = '', text = '') {
|
||
if (this.interactionVideo === videoFile) {
|
||
// 确保缓冲视频已经准备好并且可见
|
||
// this.recordedVideoBuffer.style.opacity = "1";
|
||
// 使用 zIndex 层叠，新视频在上层
|
||
this.recordedVideoBuffer.style.zIndex = "2";
|
||
this.recordedVideo.style.zIndex = "1";
|
||
|
||
// 延迟隐藏下层视频，确保无缝切换
|
||
// setTimeout(() => {
|
||
// this.recordedVideo.style.opacity = "0";
|
||
// }, 100);
|
||
} else {
|
||
// 确保主视频已经准备好并且可见
|
||
// this.recordedVideo.style.opacity = "1";
|
||
// 使用 zIndex 层叠，主视频在上层
|
||
this.recordedVideo.style.zIndex = "2";
|
||
this.recordedVideoBuffer.style.zIndex = "1";
|
||
|
||
// 延迟隐藏下层视频，确保无缝切换
|
||
// setTimeout(() => {
|
||
// this.recordedVideoBuffer.style.opacity = "0";
|
||
// }, 100);
|
||
}
|
||
}
|
||
|
||
// 添加视频帧缓存方法
|
||
captureVideoFrame(videoElement) {
|
||
const canvas = document.createElement('canvas');
|
||
const ctx = canvas.getContext('2d');
|
||
|
||
canvas.width = videoElement.videoWidth || videoElement.clientWidth;
|
||
canvas.height = videoElement.videoHeight || videoElement.clientHeight;
|
||
|
||
// 绘制当前视频帧到canvas
|
||
ctx.drawImage(videoElement, 0, 0, canvas.width, canvas.height);
|
||
|
||
return canvas.toDataURL('image/jpeg', 0.8);
|
||
}
|
||
|
||
// 创建静态帧显示元素
|
||
createFrameOverlay(frameData) {
|
||
const overlay = document.createElement('div');
|
||
overlay.style.position = 'absolute';
|
||
overlay.style.top = '0';
|
||
overlay.style.left = '0';
|
||
overlay.style.width = '100%';
|
||
overlay.style.height = '100%';
|
||
overlay.style.backgroundImage = `url(${frameData})`;
|
||
overlay.style.backgroundSize = 'cover';
|
||
overlay.style.backgroundPosition = 'center';
|
||
overlay.style.backgroundRepeat = 'no-repeat';
|
||
overlay.style.zIndex = '10';
|
||
overlay.style.pointerEvents = 'none';
|
||
overlay.style.willChange = 'auto'; // 优化渲染性能
|
||
overlay.id = 'video-frame-overlay';
|
||
|
||
return overlay;
|
||
}
|
||
|
||
// 新增平滑视频切换方法 - 等待完全传输后切换
|
||
async switchVideoStreamSmooth(videoFile, type = '', text = '') {
|
||
try {
|
||
this.logMessage(`开始等待视频流传输完成: ${videoFile} (${type})`, 'info');
|
||
|
||
// 1. 获取当前和缓冲视频元素
|
||
const currentVideo = this.activeVideoElement === 'main' ? this.recordedVideo : this.recordedVideoBuffer;
|
||
const bufferVideo = this.activeVideoElement === 'main' ? this.recordedVideoBuffer : this.recordedVideo;
|
||
|
||
// 2. 检查是否已缓存
|
||
const isCached = this.videoStreams.has(videoFile);
|
||
if (!isCached) {
|
||
this.showVideoLoading();
|
||
}
|
||
|
||
// 3. 准备新视频流
|
||
let newStream;
|
||
if (isCached) {
|
||
const cachedStream = this.videoStreams.get(videoFile);
|
||
if (cachedStream && cachedStream.getTracks().length > 0) {
|
||
newStream = cachedStream;
|
||
this.logMessage(`使用缓存视频流: ${videoFile}`, 'success');
|
||
} else {
|
||
newStream = await this.createVideoStream(videoFile);
|
||
}
|
||
} else {
|
||
newStream = await this.createVideoStream(videoFile);
|
||
}
|
||
|
||
// 4. 在缓冲视频元素中预加载新视频
|
||
bufferVideo.srcObject = newStream;
|
||
|
||
// 5. 等待视频流完全传输和准备就绪
|
||
await new Promise((resolve, reject) => {
|
||
const timeout = setTimeout(() => {
|
||
reject(new Error('视频流传输超时'));
|
||
}, 15000); // 增加超时时间到15秒，确保有足够时间传输
|
||
|
||
let frameCount = 0;
|
||
const minFrames = 10; // 增加最小帧数确保传输完整
|
||
let isStreamReady = false;
|
||
|
||
const onReady = () => {
|
||
clearTimeout(timeout);
|
||
bufferVideo.removeEventListener('canplay', onCanPlay);
|
||
bufferVideo.removeEventListener('canplaythrough', onCanPlayThrough);
|
||
bufferVideo.removeEventListener('timeupdate', onTimeUpdate);
|
||
bufferVideo.removeEventListener('error', onError);
|
||
bufferVideo.removeEventListener('loadeddata', onLoadedData);
|
||
resolve();
|
||
};
|
||
|
||
// 检查视频数据是否完全加载
|
||
const onLoadedData = () => {
|
||
this.logMessage(`视频数据开始加载: ${videoFile}`, 'info');
|
||
};
|
||
|
||
// 检查视频是否可以流畅播放（数据传输充足）
|
||
const onCanPlayThrough = () => {
|
||
this.logMessage(`视频流传输充足，可以流畅播放: ${videoFile}`, 'success');
|
||
isStreamReady = true;
|
||
|
||
// 开始播放并等待稳定帧
|
||
bufferVideo.play().then(() => {
|
||
const checkFrameStability = () => {
|
||
if (bufferVideo.currentTime > 0) {
|
||
frameCount++;
|
||
if (frameCount >= minFrames && isStreamReady) {
|
||
// 额外等待确保传输稳定
|
||
setTimeout(() => {
|
||
onReady();
|
||
}, 200); // 等待200ms确保传输稳定
|
||
} else {
|
||
requestAnimationFrame(checkFrameStability);
|
||
}
|
||
} else {
|
||
requestAnimationFrame(checkFrameStability);
|
||
}
|
||
};
|
||
requestAnimationFrame(checkFrameStability);
|
||
}).catch(reject);
|
||
};
|
||
|
||
// 基本播放准备就绪
|
||
const onCanPlay = () => {
|
||
if (!isStreamReady) {
|
||
this.logMessage(`视频基本准备就绪，等待完全传输: ${videoFile}`, 'info');
|
||
// 如果还没有canplaythrough事件，继续等待
|
||
}
|
||
};
|
||
|
||
// 时间更新监听（备用检查）
|
||
const onTimeUpdate = () => {
|
||
if (bufferVideo.currentTime > 0 && isStreamReady) {
|
||
frameCount++;
|
||
if (frameCount >= minFrames) {
|
||
setTimeout(() => {
|
||
onReady();
|
||
}, 200);
|
||
}
|
||
}
|
||
};
|
||
|
||
const onError = (error) => {
|
||
clearTimeout(timeout);
|
||
bufferVideo.removeEventListener('canplay', onCanPlay);
|
||
bufferVideo.removeEventListener('canplaythrough', onCanPlayThrough);
|
||
bufferVideo.removeEventListener('timeupdate', onTimeUpdate);
|
||
bufferVideo.removeEventListener('error', onError);
|
||
bufferVideo.removeEventListener('loadeddata', onLoadedData);
|
||
reject(error);
|
||
};
|
||
|
||
// 检查当前状态
|
||
if (bufferVideo.readyState >= 4) { // HAVE_ENOUGH_DATA
|
||
onCanPlayThrough();
|
||
} else {
|
||
bufferVideo.addEventListener('loadeddata', onLoadedData);
|
||
bufferVideo.addEventListener('canplay', onCanPlay);
|
||
bufferVideo.addEventListener('canplaythrough', onCanPlayThrough);
|
||
bufferVideo.addEventListener('timeupdate', onTimeUpdate);
|
||
bufferVideo.addEventListener('error', onError);
|
||
}
|
||
});
|
||
|
||
// 6. 最终确认视频流完全准备就绪
|
||
await new Promise(resolve => {
|
||
let confirmCount = 0;
|
||
const maxConfirms = 3;
|
||
|
||
const finalCheck = () => {
|
||
if (bufferVideo.readyState >= 4 && bufferVideo.currentTime > 0) {
|
||
confirmCount++;
|
||
if (confirmCount >= maxConfirms) {
|
||
this.logMessage(`视频流传输完成，准备切换: ${videoFile}`, 'success');
|
||
resolve();
|
||
} else {
|
||
setTimeout(finalCheck, 50); // 每50ms检查一次
|
||
}
|
||
} else {
|
||
setTimeout(finalCheck, 50);
|
||
}
|
||
};
|
||
finalCheck();
|
||
});
|
||
|
||
// 7. 执行无缝切换（当前视频继续播放直到新视频完全准备好）
|
||
bufferVideo.style.zIndex = '2';
|
||
currentVideo.style.zIndex = '1';
|
||
|
||
this.logMessage(`视频切换完成: ${videoFile}`, 'success');
|
||
|
||
// 8. 隐藏加载指示器
|
||
if (!isCached) {
|
||
this.hideVideoLoading();
|
||
}
|
||
|
||
// 9. 更新状态
|
||
this.currentVideoStream = newStream;
|
||
this.currentVideo = videoFile;
|
||
this.activeVideoElement = this.activeVideoElement === 'main' ? 'buffer' : 'main';
|
||
|
||
// 10. 延迟清理旧视频流
|
||
setTimeout(() => {
|
||
if (currentVideo.srcObject && currentVideo.srcObject !== newStream) {
|
||
currentVideo.srcObject.getTracks().forEach(track => track.stop());
|
||
currentVideo.srcObject = null;
|
||
}
|
||
}, 1000);
|
||
|
||
// 11. 更新WebRTC连接
|
||
if (this.peerConnection && this.videoSender) {
|
||
const newVideoTrack = newStream.getVideoTracks()[0];
|
||
if (newVideoTrack) {
|
||
await this.videoSender.replaceTrack(newVideoTrack);
|
||
}
|
||
}
|
||
|
||
// 12. 更新显示信息
|
||
if (text) {
|
||
this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`;
|
||
this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success');
|
||
} else {
|
||
this.currentVideoName.textContent = `视频流: ${videoFile}`;
|
||
this.logMessage(`成功切换到视频流: ${videoFile}`, 'success');
|
||
}
|
||
|
||
} catch (error) {
|
||
this.logMessage(`视频流切换失败: ${error.message}`, 'error');
|
||
|
||
this.hideVideoLoading();
|
||
|
||
// 回退到默认视频
|
||
if (videoFile !== this.defaultVideo) {
|
||
this.logMessage('尝试回到默认视频', 'info');
|
||
await this.switchVideoStreamSmooth(this.defaultVideo, 'fallback');
|
||
}
|
||
}
|
||
}
|
||
|
||
|
||
|
||
// 显示加载指示器
|
||
showVideoLoading() {
|
||
if (this.videoLoading) {
|
||
this.videoLoading.classList.add('show');
|
||
}
|
||
}
|
||
|
||
// 隐藏加载指示器
|
||
hideVideoLoading() {
|
||
if (this.videoLoading) {
|
||
this.videoLoading.classList.remove('show');
|
||
}
|
||
}
|
||
|
||
bindEvents() {
|
||
// 开始通话按钮 - 添加视频准备状态检查
|
||
this.startButton.onclick = () => {
|
||
if (!this.isVideoReady) {
|
||
this.logMessage('还在加载中，请稍候...', 'warning');
|
||
return;
|
||
}
|
||
this.startCall();
|
||
};
|
||
|
||
// 停止通话按钮 - 改为调用 userDisconnect
|
||
this.stopButton.onclick = () => this.userDisconnect();
|
||
|
||
// 静音按钮
|
||
// this.muteButton.onclick = () => this.toggleMute();
|
||
|
||
// 回到默认视频按钮
|
||
// this.defaultVideoButton.onclick = () => this.returnToDefaultVideo();
|
||
|
||
// 测试视频文件按钮
|
||
// this.testVideoButton.onclick = () => this.testAllVideoFiles();
|
||
|
||
// 发送文本按钮
|
||
this.sendTextButton.onclick = () => this.sendText();
|
||
|
||
// 回车键发送文本
|
||
this.textInput.onkeypress = (e) => {
|
||
if (e.key === 'Enter') {
|
||
this.sendText();
|
||
}
|
||
};
|
||
|
||
// 语音输入按钮
|
||
// this.startVoiceButton.onclick = () => this.startVoiceRecording();
|
||
// this.stopVoiceButton.onclick = () => this.stopVoiceRecording();
|
||
}
|
||
|
||
// 显示等待连接提示
|
||
showConnectionWaiting() {
|
||
if (this.connectionWaiting) {
|
||
this.connectionWaiting.style.display = 'block';
|
||
// 使用setTimeout确保display设置后再添加show类
|
||
setTimeout(() => {
|
||
this.connectionWaiting.classList.add('show');
|
||
}, 10);
|
||
}
|
||
}
|
||
|
||
// 隐藏等待连接提示
|
||
hideConnectionWaiting() {
|
||
if (this.connectionWaiting) {
|
||
this.connectionWaiting.classList.remove('show');
|
||
this.connectionWaiting.style.display = 'none';
|
||
// // 等待动画完成后隐藏元素
|
||
// setTimeout(() => {
|
||
// this.connectionWaiting.style.display = 'none';
|
||
// }, 300);
|
||
}
|
||
}
|
||
|
||
async startCall() {
|
||
try {
|
||
// 检查所有必要条件
|
||
if (!this.isVideoReady || !this.isDefaultVideoLoaded) {
|
||
this.logMessage('视频资源尚未准备就绪，请稍候...', 'warning');
|
||
return;
|
||
}
|
||
|
||
if (!this.socket || !this.socket.connected) {
|
||
this.logMessage('网络连接未就绪，请稍候...', 'warning');
|
||
return;
|
||
}
|
||
|
||
// 隐藏开始通话按钮
|
||
this.startButton.style.display = 'none';
|
||
// 显示等待连接提示
|
||
this.showConnectionWaiting();
|
||
// 切换到通话中图标
|
||
this.switchToCallingIcon();
|
||
|
||
// 现在才开始显示视频
|
||
await this.startDefaultVideoStream();
|
||
|
||
// 添加更详细的错误处理
|
||
console.log('开始请求麦克风权限...');
|
||
this.localStream = await navigator.mediaDevices.getUserMedia({
|
||
video: false,
|
||
audio: true
|
||
});
|
||
console.log('麦克风权限获取成功');
|
||
|
||
await this.createPeerConnection();
|
||
await this.startVoiceRecording();
|
||
|
||
this.startButton.disabled = true;
|
||
this.startButton.style.opacity = '0.5'
|
||
this.stopButton.disabled = false;
|
||
|
||
// 隐藏头像，显示视频
|
||
if (this.videoContainer) {
|
||
|
||
this.videoContainer.classList.add('calling');
|
||
}
|
||
|
||
// 显示结束通话按钮
|
||
this.stopButton.style.display = 'block';
|
||
|
||
|
||
|
||
this.updateAudioStatus('已连接', 'connected');
|
||
this.logMessage('音频通话已开始', 'success');
|
||
|
||
// 确保视频映射已加载
|
||
if (Object.keys(this.videoMapping).length === 0) {
|
||
await this.loadVideoMapping();
|
||
}
|
||
|
||
this.logMessage(`视频映射已加载: ${Object.keys(this.videoMapping).length} 个映射`, 'info');
|
||
|
||
// 通知服务器通话开始
|
||
this.socket.emit('call-started');
|
||
|
||
// 开始播放当前场景的默认视频
|
||
// await this.precreateImportantVideos();
|
||
|
||
|
||
|
||
} catch (error) {
|
||
this.logMessage(`开始通话失败: ${error.message}`, 'error');
|
||
// 恢复开始通话按钮
|
||
this.startButton.style.display = 'block';
|
||
// 如果出错，隐藏等待连接提示并恢复到默认图标
|
||
this.hideConnectionWaiting();
|
||
this.switchToDefaultIcon();
|
||
}
|
||
}
|
||
|
||
stopCall() {
|
||
// 隐藏等待连接提示
|
||
this.hideConnectionWaiting();
|
||
// 恢复到默认图标
|
||
this.switchToDefaultIcon();
|
||
|
||
// 只有用户主动点击关闭通话时才发送断开事件
|
||
// 移除自动发送 user-disconnect 事件
|
||
// if (this.socket && this.socket.connected) {
|
||
// this.socket.emit('user-disconnect');
|
||
// }
|
||
|
||
// 停止音频处理器
|
||
if (this.audioProcessor) {
|
||
this.audioProcessor.stopRecording();
|
||
}
|
||
|
||
if (this.localStream) {
|
||
this.localStream.getTracks().forEach(track => {
|
||
track.stop();
|
||
console.log(`停止轨道: ${track.kind}`);
|
||
});
|
||
this.localStream = null;
|
||
}
|
||
|
||
if (this.peerConnection) {
|
||
this.peerConnection.close();
|
||
this.peerConnection = null;
|
||
}
|
||
|
||
// 隐藏结束通话按钮
|
||
this.stopButton.style.display = 'none';
|
||
this.stopButton.disabled = true;
|
||
|
||
// 隐藏视频元素
|
||
if (this.recordedVideo) {
|
||
this.recordedVideo.style.display = 'none';
|
||
this.recordedVideo.srcObject = null;
|
||
}
|
||
if (this.recordedVideoBuffer) {
|
||
this.recordedVideoBuffer.style.display = 'none';
|
||
this.recordedVideoBuffer.srcObject = null;
|
||
}
|
||
|
||
// 显示开始通话按钮
|
||
this.startButton.disabled = true;
|
||
this.startButton.style.display = 'block';
|
||
this.startButton.style.opacity = '0.5';
|
||
|
||
// 移除页面刷新，保持websocket连接
|
||
// setTimeout(() => {
|
||
// window.location.reload();
|
||
// }, 300);
|
||
|
||
// 清理视频缓存和预创建流
|
||
this.clearVideoCache();
|
||
|
||
// setTimeout(() => {
|
||
// // 显示头像，隐藏视频
|
||
// if (this.videoContainer) {
|
||
// this.videoContainer.classList.remove('calling');
|
||
// }
|
||
|
||
// // 重新初始化重要视频流
|
||
// this.precreateImportantVideos().then(() => {
|
||
// // 重新启动默认视频流
|
||
// this.startDefaultVideoStream();
|
||
// });
|
||
// }, 300);
|
||
}
|
||
|
||
// 新增：用户主动断开连接的方法
|
||
userDisconnect() {
|
||
// 发送用户关闭连接事件到后端
|
||
if (this.socket && this.socket.connected) {
|
||
this.socket.emit('user-disconnect');
|
||
|
||
// 等待服务器确认断开后再刷新
|
||
this.socket.on('disconnect', () => {
|
||
console.log('WebSocket已断开，准备刷新页面...');
|
||
setTimeout(() => {
|
||
window.location.reload();
|
||
}, 500);
|
||
});
|
||
|
||
// 主动断开连接
|
||
setTimeout(() => {
|
||
this.socket.disconnect();
|
||
}, 200);
|
||
|
||
// 兜底机制：如果2秒内没有正常断开，强制刷新
|
||
setTimeout(() => {
|
||
if (this.socket && this.socket.connected) {
|
||
console.log('WebSocket断开超时，强制刷新页面');
|
||
window.location.reload();
|
||
}
|
||
}, 2000);
|
||
} else {
|
||
// 如果socket已经断开，直接刷新
|
||
setTimeout(() => {
|
||
window.location.reload();
|
||
}, 100);
|
||
}
|
||
|
||
// 调用停止通话
|
||
this.stopCall();
|
||
}
|
||
|
||
// 清除视频缓存的方法
|
||
clearVideoCache() {
|
||
// 清除视频元素缓存
|
||
if (this.recordedVideo) {
|
||
this.recordedVideo.src = '';
|
||
this.recordedVideo.srcObject = null;
|
||
this.recordedVideo.load(); // 重新加载空视频
|
||
// 暂停视频播放
|
||
this.recordedVideo.pause();
|
||
}
|
||
|
||
if (this.recordedVideoBuffer) {
|
||
this.recordedVideoBuffer.src = '';
|
||
this.recordedVideoBuffer.srcObject = null;
|
||
this.recordedVideoBuffer.load();
|
||
this.recordedVideoBuffer.pause();
|
||
}
|
||
|
||
// 清除视频流缓存
|
||
if (this.currentVideoStream) {
|
||
this.currentVideoStream.getTracks().forEach(track => track.stop());
|
||
this.currentVideoStream = null;
|
||
}
|
||
|
||
// 清除视频流映射缓存
|
||
this.videoStreams.clear();
|
||
|
||
// 清除预创建的视频流
|
||
if (this.precreatedStreams) {
|
||
this.precreatedStreams.forEach((stream, videoFile) => {
|
||
if (stream) {
|
||
stream.getTracks().forEach(track => track.stop());
|
||
}
|
||
});
|
||
this.precreatedStreams.clear();
|
||
}
|
||
|
||
// 重置初始化状态，允许重新预创建
|
||
this.isInitialized = false;
|
||
|
||
// 重置视频相关状态
|
||
this.currentVideo = null;
|
||
this.activeVideoElement = 'main';
|
||
this.videoSender = null;
|
||
|
||
this.logMessage('视频缓存已清除，视频已停止', 'info');
|
||
}
|
||
|
||
// 添加清除API缓存的方法
|
||
async clearApiCache() {
|
||
// 重新加载视频映射
|
||
this.videoMapping = {};
|
||
await this.loadVideoMapping();
|
||
|
||
// 重新加载默认视频
|
||
await this.loadDefaultVideo();
|
||
|
||
// 重新加载视频列表
|
||
await this.loadVideoList();
|
||
|
||
this.logMessage('API缓存已清除并重新加载', 'info');
|
||
}
|
||
|
||
async createPeerConnection() {
|
||
const configuration = {
|
||
iceServers: [
|
||
{ urls: "stun:stun.qq.com:3478" },
|
||
{ urls: "stun:stun.miwifi.com:3478" },
|
||
{ urls: 'stun:stun.l.google.com:19302' },
|
||
{ urls: 'stun:stun1.l.google.com:19302' }
|
||
]
|
||
};
|
||
|
||
this.peerConnection = new RTCPeerConnection(configuration);
|
||
|
||
// 添加本地音频流
|
||
this.localStream.getTracks().forEach(track => {
|
||
this.peerConnection.addTrack(track, this.localStream);
|
||
});
|
||
|
||
// 移除视频轨道添加逻辑，只使用音频进行WebRTC通信
|
||
// 视频将直接在本地切换，不通过WebRTC传输
|
||
|
||
// 处理远程流
|
||
this.peerConnection.ontrack = (event) => {
|
||
this.remoteVideo.srcObject = event.streams[0];
|
||
};
|
||
|
||
// 处理ICE候选
|
||
this.peerConnection.onicecandidate = (event) => {
|
||
if (event.candidate) {
|
||
this.socket.emit('ice-candidate', {
|
||
candidate: event.candidate
|
||
});
|
||
}
|
||
};
|
||
|
||
// 创建并发送offer
|
||
this.peerConnection.createOffer()
|
||
.then(offer => this.peerConnection.setLocalDescription(offer))
|
||
.then(() => {
|
||
this.socket.emit('offer', {
|
||
offer: this.peerConnection.localDescription
|
||
});
|
||
})
|
||
.catch(error => {
|
||
this.logMessage('创建offer失败: ' + error.message, 'error');
|
||
});
|
||
}
|
||
|
||
async handleOffer(data) {
|
||
if (!this.peerConnection) {
|
||
await this.startCall();
|
||
}
|
||
|
||
await this.peerConnection.setRemoteDescription(new RTCSessionDescription(data.offer));
|
||
|
||
const answer = await this.peerConnection.createAnswer();
|
||
await this.peerConnection.setLocalDescription(answer);
|
||
|
||
this.socket.emit('answer', {
|
||
answer: this.peerConnection.localDescription
|
||
});
|
||
}
|
||
|
||
async handleAnswer(data) {
|
||
if (this.peerConnection) {
|
||
await this.peerConnection.setRemoteDescription(new RTCSessionDescription(data.answer));
|
||
}
|
||
}
|
||
|
||
async handleIceCandidate(data) {
|
||
if (this.peerConnection) {
|
||
await this.peerConnection.addIceCandidate(new RTCIceCandidate(data.candidate));
|
||
}
|
||
}
|
||
|
||
toggleMute() {
|
||
if (this.localStream) {
|
||
const audioTrack = this.localStream.getAudioTracks()[0];
|
||
if (audioTrack) {
|
||
audioTrack.enabled = !audioTrack.enabled;
|
||
this.muteButton.textContent = audioTrack.enabled ? '静音' : '取消静音';
|
||
this.logMessage(audioTrack.enabled ? '已取消静音' : '已静音', 'info');
|
||
}
|
||
}
|
||
}
|
||
|
||
async sendText() {
|
||
const text = this.textInput.value.trim();
|
||
// const text = 'say-5s-m-sw';
|
||
if (text) {
|
||
this.socket.emit('text-input', { text });
|
||
this.logMessage(`发送文本: ${text}`, 'info');
|
||
this.textInput.value = '';
|
||
|
||
try {
|
||
// 调用chat_with_audio进行大模型回答和音频合成
|
||
this.logMessage('正在处理文本，请稍候...', 'info');
|
||
const result = await chatWithAudioStream(text);
|
||
this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
|
||
|
||
// // // 根据文本查找对应视频并切换
|
||
// await this.handleTextInput(text);
|
||
|
||
// 并行执行两个操作
|
||
// const [result] = await Promise.all([
|
||
// this.handleTextInput(text),
|
||
// chatWithAudioStream(text)
|
||
// // 视频切换可以立即开始
|
||
// ]);
|
||
|
||
// this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
|
||
} catch (error) {
|
||
this.logMessage(`处理文本失败: ${error.message}`, 'error');
|
||
console.error('chatWithAudioStream error:', error);
|
||
}
|
||
}
|
||
}
|
||
|
||
async handleTextInput(text) {
|
||
// 根据文本查找对应视频
|
||
let videoFile = this.videoMapping['default'] || this.defaultVideo;
|
||
for (const [key, value] of Object.entries(this.videoMapping)) {
|
||
if (text.toLowerCase().includes(key.toLowerCase())) {
|
||
videoFile = value;
|
||
break;
|
||
}
|
||
}
|
||
|
||
// 检查当前视频播放状态
|
||
const currentVideo = this.recordedVideo;
|
||
const isVideoPlaying = !currentVideo.paused && !currentVideo.ended && currentVideo.currentTime > 0;
|
||
|
||
if (isVideoPlaying && this.currentVideo !== videoFile) {
|
||
// 如果当前视频正在播放且需要切换到不同视频
|
||
// 可以选择立即切换或等待当前视频播放完成
|
||
console.log('当前视频正在播放，准备切换到:', videoFile);
|
||
|
||
// 立即切换（推荐）
|
||
await this.switchVideoStream(videoFile, 'text', text);
|
||
|
||
// 或者等待当前视频播放完成再切换（可选）
|
||
// await this.waitForVideoToFinish();
|
||
// await this.switchVideoStream(videoFile, 'text', text);
|
||
} else {
|
||
// 直接切换
|
||
await this.switchVideoStream(videoFile, 'text', text);
|
||
}
|
||
|
||
// 通知服务器切换视频流
|
||
this.socket.emit('switch-video-stream', {
|
||
videoFile,
|
||
type: 'text',
|
||
text
|
||
});
|
||
}
|
||
|
||
// 修改：使用音频处理器的语音录制功能
|
||
async startVoiceRecording() {
|
||
const success = await this.audioProcessor.startRecording(this.localStream);
|
||
|
||
if (success) {
|
||
this.logMessage('高级语音录制已启动', 'success');
|
||
} else {
|
||
this.logMessage('录音启动失败', 'error');
|
||
}
|
||
}
|
||
|
||
// 修改：停止语音录制
|
||
stopVoiceRecording() {
|
||
this.audioProcessor.stopRecording();
|
||
|
||
// this.startVoiceButton.disabled = false;
|
||
// this.stopVoiceButton.disabled = true;
|
||
// this.startVoiceButton.classList.remove('recording');
|
||
// this.voiceStatus.textContent = '点击开始语音输入';
|
||
|
||
this.logMessage('语音录制已停止', 'info');
|
||
}
|
||
|
||
// 处理语音输入结果
|
||
async handleVoiceInput(text) {
|
||
if(text == ""){
|
||
console.log("识别到用户未说话")
|
||
return
|
||
}
|
||
|
||
// 根据文本查找对应视频
|
||
let videoFile = this.videoMapping['default'] || this.defaultVideo;
|
||
for (const [key, value] of Object.entries(this.videoMapping)) {
|
||
if (text.toLowerCase().includes(key.toLowerCase())) {
|
||
videoFile = value;
|
||
break;
|
||
}
|
||
}
|
||
|
||
// 切换到对应的视频流
|
||
await this.switchVideoStream(videoFile, 'voice', text);
|
||
|
||
// 通知服务器切换视频流
|
||
this.socket.emit('switch-video-stream', {
|
||
videoFile,
|
||
type: 'voice',
|
||
text
|
||
});
|
||
|
||
// 调用大模型处理
|
||
try {
|
||
this.logMessage('正在处理语音输入，请稍候...', 'info', this.currentVideoTag);
|
||
// if (this.currentVideoName === "default"){
|
||
const result = await chatWithAudioStream(text);
|
||
this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
|
||
// }
|
||
|
||
} catch (error) {
|
||
this.logMessage(`处理语音输入失败: ${error.message}`, 'error');
|
||
console.error('chatWithAudioStream error:', error);
|
||
}
|
||
}
|
||
|
||
// 删除原有的简单音频处理方法
|
||
// processVoiceInput() 和 simulateSpeechRecognition() 方法已被移除
|
||
|
||
simulateSpeechRecognition() {
|
||
// 模拟语音识别，随机返回预设的文本
|
||
const texts = ['你好', '再见', '谢谢', 'hello', 'goodbye', 'thank you'];
|
||
return texts[Math.floor(Math.random() * texts.length)];
|
||
}
|
||
|
||
returnToDefaultVideo() {
|
||
this.switchVideoStream(this.defaultVideo, 'default');
|
||
this.socket.emit('return-to-default');
|
||
this.logMessage(`已返回至默认视频: ${this.defaultVideo}`, 'info');
|
||
}
|
||
|
||
updateStatus(message, type) {
|
||
this.connectionStatus.textContent = message;
|
||
this.connectionStatus.className = `status ${type}`;
|
||
// 显示状态元素（仅在连接时显示）
|
||
if (type === 'connected') {
|
||
this.connectionStatus.style.display = 'block';
|
||
}
|
||
}
|
||
|
||
updateAudioStatus(message, type) {
|
||
this.audioStatus.textContent = message;
|
||
this.audioStatus.className = `status-indicator ${type}`;
|
||
}
|
||
|
||
logMessage(message, type = 'info') {
|
||
const logEntry = document.createElement('div');
|
||
logEntry.className = type;
|
||
logEntry.textContent = `[${new Date().toLocaleTimeString()}] ${message}`;
|
||
|
||
this.messageLog.appendChild(logEntry);
|
||
this.messageLog.scrollTop = this.messageLog.scrollHeight;
|
||
}
|
||
|
||
checkVideoStreamStatus() {
|
||
const status = {
|
||
hasStream: !!this.currentVideoStream,
|
||
streamTracks: this.currentVideoStream ? this.currentVideoStream.getTracks().length : 0,
|
||
videoReadyState: this.recordedVideo.readyState,
|
||
videoPaused: this.recordedVideo.paused,
|
||
videoCurrentTime: this.recordedVideo.currentTime,
|
||
videoDuration: this.recordedVideo.duration,
|
||
currentVideo: this.currentVideo
|
||
};
|
||
|
||
this.logMessage(`视频流状态: ${JSON.stringify(status)}`, 'info');
|
||
return status;
|
||
}
|
||
|
||
// 添加图标切换方法
|
||
switchToCallingIcon() {
|
||
const callIcon = document.getElementById('callIcon');
|
||
const callingText = document.getElementById('callingText');
|
||
const startButton = this.startButton;
|
||
|
||
if (callIcon && callingText && startButton) {
|
||
callIcon.style.display = 'none';
|
||
callingText.style.display = 'block';
|
||
startButton.classList.add('calling');
|
||
startButton.title = '通话中...';
|
||
}
|
||
}
|
||
|
||
switchToDefaultIcon() {
|
||
const callIcon = document.getElementById('callIcon');
|
||
const callingText = document.getElementById('callingText');
|
||
const startButton = this.startButton;
|
||
|
||
if (callIcon && callingText && startButton) {
|
||
callIcon.style.display = 'block';
|
||
callingText.style.display = 'none';
|
||
startButton.classList.remove('calling');
|
||
startButton.title = '开始通话';
|
||
startButton.disabled = false;
|
||
}
|
||
}
|
||
|
||
async testAllVideoFiles() {
|
||
this.logMessage('开始测试所有视频文件...', 'info');
|
||
|
||
const videoFiles = ['asd.mp4', 'zxc.mp4', 'jkl.mp4'];
|
||
|
||
for (const videoFile of videoFiles) {
|
||
try {
|
||
await this.testVideoFile(videoFile);
|
||
} catch (error) {
|
||
this.logMessage(`视频文件 ${videoFile} 测试失败: ${error.message}`, 'error');
|
||
}
|
||
}
|
||
|
||
this.logMessage('视频文件测试完成', 'info');
|
||
|
||
// 检查当前视频流状态
|
||
this.checkVideoStreamStatus();
|
||
}
|
||
}
|
||
|
||
// 页面加载完成后初始化应用
|
||
document.addEventListener('DOMContentLoaded', () => {
|
||
console.log('DOMContentLoaded 事件触发');
|
||
try {
|
||
new WebRTCChat();
|
||
} catch (error) {
|
||
console.error('WebRTCChat 初始化失败:', error);
|
||
}
|
||
});
|