// WebRTC 音视频通话应用 import { chatWithAudioStream } from './chat_with_audio.js'; import { AudioProcessor } from './audio_processor.js'; class WebRTCChat { constructor() { this.socket = null; this.localStream = null; this.peerConnection = null; this.isRecording = false; this.mediaRecorder = null; this.audioChunks = []; this.videoMapping = {}; this.defaultVideo = 'asd.mp4'; this.currentVideo = null; this.videoStreams = new Map(); // 存储不同视频的MediaStream this.currentVideoStream = null; // 初始化音频处理器 this.audioProcessor = new AudioProcessor({ onSpeechStart: () => { this.voiceStatus.textContent = '检测到语音,开始录音...'; this.logMessage('检测到语音,开始录音...', 'info'); }, onSpeechEnd: () => { // 语音结束回调 }, onRecognitionResult: (text) => { // ASRTEXT = text; this.voiceStatus.textContent = '识别完成'; this.logMessage(`语音识别结果: ${text}`, 'success'); this.handleVoiceInput(text); }, onError: (error) => { this.voiceStatus.textContent = '识别失败'; this.logMessage(error, 'error'); }, onStatusUpdate: (message, status) => { this.voiceStatus.textContent = message; } }); this.initializeElements(); this.initializeSocket(); this.loadVideoMapping(); this.loadVideoList(); this.loadDefaultVideo(); this.bindEvents(); } initializeElements() { // 视频元素 this.localVideo = document.getElementById('localVideo'); this.remoteVideo = document.getElementById('remoteVideo'); this.recordedVideo = document.getElementById('recordedVideo'); // 音频状态元素 this.audioStatus = document.getElementById('audioStatus'); // 按钮元素 this.startButton = document.getElementById('startButton'); this.stopButton = document.getElementById('stopButton'); this.muteButton = document.getElementById('muteButton'); this.sendTextButton = document.getElementById('sendTextButton'); this.startVoiceButton = document.getElementById('startVoiceButton'); this.stopVoiceButton = document.getElementById('stopVoiceButton'); this.defaultVideoButton = document.getElementById('defaultVideoButton'); this.testVideoButton = document.getElementById('testVideoButton'); // 新增测试按钮 // 输入元素 this.textInput = document.getElementById('textInput'); this.voiceStatus = document.getElementById('voiceStatus'); // 状态元素 this.connectionStatus = document.getElementById('connectionStatus'); this.messageLog = document.getElementById('messageLog'); this.currentVideoName = document.getElementById('currentVideoName'); this.videoList = document.getElementById('videoList'); } initializeSocket() { this.socket = io(); this.socket.on('connect', () => { this.updateStatus('已连接到服务器', 'connected'); this.logMessage('已连接到服务器', 'success'); }); this.socket.on('disconnect', () => { this.updateStatus('与服务器断开连接', 'disconnected'); this.logMessage('与服务器断开连接', 'error'); }); // WebRTC 信令处理 this.socket.on('offer', (data) => { this.handleOffer(data); }); this.socket.on('answer', (data) => { this.handleAnswer(data); }); this.socket.on('ice-candidate', (data) => { this.handleIceCandidate(data); }); // 视频流切换处理 this.socket.on('video-stream-switched', (data) => { this.logMessage(`收到视频流切换指令: ${data.videoFile} (${data.type}) 来自用户: ${data.from}`, 'info'); this.switchVideoStream(data.videoFile, data.type, data.text); }); // 通话开始处理 this.socket.on('call-started', (data) => { this.logMessage('通话已开始', 'success'); this.startDefaultVideoStream(); }); } async loadVideoMapping() { try { const response = await fetch('/api/video-mapping'); const data = await response.json(); this.videoMapping = data.mapping; this.logMessage('视频映射加载成功', 'success'); } catch (error) { this.logMessage('加载视频映射失败: ' + error.message, 'error'); } } async loadDefaultVideo() { try { const response = await fetch('/api/default-video'); const data = await response.json(); this.defaultVideo = data.defaultVideo; this.logMessage('默认视频配置加载成功', 'success'); } catch (error) { this.logMessage('加载默认视频配置失败: ' + error.message, 'error'); } } async loadVideoList() { try { const response = await fetch('/api/videos'); const data = await response.json(); this.renderVideoList(data.videos); this.logMessage('视频列表加载成功', 'success'); } catch (error) { this.logMessage('加载视频列表失败: ' + error.message, 'error'); } } renderVideoList(videos) { this.videoList.innerHTML = ''; videos.forEach(video => { const videoItem = document.createElement('div'); videoItem.className = 'video-item'; videoItem.textContent = video; videoItem.onclick = () => this.selectVideo(video); this.videoList.appendChild(videoItem); }); } selectVideo(videoFile) { // 移除之前的active类 document.querySelectorAll('.video-item').forEach(item => { item.classList.remove('active'); }); // 添加active类到选中的视频 event.target.classList.add('active'); // 切换到选中的视频流 this.switchVideoStream(videoFile, 'manual'); // 通知服务器切换视频流 this.socket.emit('switch-video-stream', { videoFile, type: 'manual' }); } async startDefaultVideoStream() { try { this.logMessage('开始创建默认视频流', 'info'); // 添加加载状态 this.recordedVideo.classList.add('loading'); // 创建默认视频的MediaStream const defaultStream = await this.createVideoStream(this.defaultVideo); // 等待流稳定 await new Promise(resolve => setTimeout(resolve, 500)); // 检查流是否有效 if (!defaultStream || defaultStream.getTracks().length === 0) { throw new Error('默认视频流创建失败'); } // 设置视频流 this.currentVideoStream = defaultStream; this.recordedVideo.srcObject = defaultStream; this.currentVideo = this.defaultVideo; this.currentVideoName.textContent = `默认视频: ${this.defaultVideo}`; // 等待视频元素准备就绪 await new Promise(resolve => { const checkReady = () => { if (this.recordedVideo.readyState >= 2) { // HAVE_CURRENT_DATA resolve(); } else { setTimeout(checkReady, 100); } }; checkReady(); }); // 确保视频开始播放 try { await this.recordedVideo.play(); this.logMessage('默认视频开始播放', 'success'); // 移除加载状态,添加播放状态 this.recordedVideo.classList.remove('loading'); this.recordedVideo.classList.add('playing'); } catch (playError) { this.logMessage(`默认视频播放失败: ${playError.message}`, 'error'); this.recordedVideo.classList.remove('loading'); } this.logMessage('默认视频流创建成功', 'success'); } catch (error) { this.logMessage('创建默认视频流失败: ' + error.message, 'error'); this.recordedVideo.classList.remove('loading'); } } async testVideoFile(videoFile) { return new Promise((resolve, reject) => { const testVideo = document.createElement('video'); testVideo.src = `/videos/${videoFile}`; testVideo.muted = true; testVideo.onloadedmetadata = () => { this.logMessage(`视频文件测试成功: ${videoFile} (${testVideo.videoWidth}x${testVideo.videoHeight})`, 'success'); resolve(true); }; testVideo.onerror = () => { this.logMessage(`视频文件测试失败: ${videoFile}`, 'error'); reject(new Error(`视频文件不存在或无法加载: ${videoFile}`)); }; // 设置超时 setTimeout(() => { reject(new Error(`视频文件加载超时: ${videoFile}`)); }, 10000); }); } async createVideoStream(videoFile) { // 如果已经缓存了这个视频流,直接返回 if (this.videoStreams.has(videoFile)) { this.logMessage(`使用缓存的视频流: ${videoFile}`, 'info'); return this.videoStreams.get(videoFile); } try { this.logMessage(`开始创建视频流: ${videoFile}`, 'info'); // 先测试视频文件是否存在 await this.testVideoFile(videoFile); // 创建video元素来加载视频 const video = document.createElement('video'); video.src = `/videos/${videoFile}`; video.muted = true; video.loop = true; video.autoplay = true; video.crossOrigin = 'anonymous'; // 添加跨域支持 video.playsInline = true; // 添加playsInline属性 // 等待视频加载完成 await new Promise((resolve, reject) => { video.onloadedmetadata = () => { this.logMessage(`视频元数据加载完成: ${videoFile}`, 'info'); resolve(); }; video.onerror = (error) => { this.logMessage(`视频加载失败: ${videoFile}`, 'error'); reject(error); }; video.onloadstart = () => { this.logMessage(`开始加载视频: ${videoFile}`, 'info'); }; video.oncanplay = () => { this.logMessage(`视频可以播放: ${videoFile}`, 'info'); }; }); // 创建MediaStream const canvas = document.createElement('canvas'); const ctx = canvas.getContext('2d'); // 设置canvas尺寸为视频尺寸 canvas.width = video.videoWidth || 640; canvas.height = video.videoHeight || 480; this.logMessage(`Canvas尺寸: ${canvas.width}x${canvas.height}`, 'info'); // 开始播放视频 try { await video.play(); this.logMessage(`视频开始播放: ${videoFile}`, 'info'); } catch (playError) { this.logMessage(`视频播放失败: ${playError.message}`, 'error'); // 即使播放失败也继续创建流 } // 等待视频开始播放 await new Promise(resolve => { const checkPlay = () => { if (video.readyState >= video.HAVE_CURRENT_DATA) { resolve(); } else { setTimeout(checkPlay, 100); } }; checkPlay(); }); // 绘制视频到canvas let isDrawing = false; const drawFrame = () => { if (video.readyState >= video.HAVE_CURRENT_DATA && !isDrawing) { isDrawing = true; ctx.drawImage(video, 0, 0, canvas.width, canvas.height); isDrawing = false; } requestAnimationFrame(drawFrame); }; // 开始绘制帧 drawFrame(); // 从canvas创建MediaStream const stream = canvas.captureStream(30); // 30fps // 等待流创建完成并稳定 await new Promise(resolve => { setTimeout(resolve, 500); // 给更多时间让流稳定 }); this.logMessage(`视频流创建成功: ${videoFile}`, 'success'); // 缓存这个视频流 this.videoStreams.set(videoFile, stream); return stream; } catch (error) { this.logMessage(`创建视频流失败 ${videoFile}: ${error.message}`, 'error'); throw error; } } async switchVideoStream(videoFile, type = '', text = '') { try { this.logMessage(`开始切换视频流: ${videoFile} (${type})`, 'info'); // 添加加载状态 this.recordedVideo.classList.add('loading'); // 先创建新的视频流,不立即停止旧的 const newStream = await this.createVideoStream(videoFile); // 等待流稳定 await new Promise(resolve => setTimeout(resolve, 200)); // 检查流是否有效 if (!newStream || newStream.getTracks().length === 0) { throw new Error('创建的视频流无效'); } // 先设置新的视频流,再停止旧的 this.currentVideoStream = newStream; this.recordedVideo.srcObject = newStream; this.currentVideo = videoFile; // 确保视频开始播放 try { await this.recordedVideo.play(); this.logMessage('视频元素开始播放', 'info'); // 移除加载状态,添加播放状态 this.recordedVideo.classList.remove('loading'); this.recordedVideo.classList.add('playing'); } catch (playError) { this.logMessage(`视频播放失败: ${playError.message}`, 'error'); this.recordedVideo.classList.remove('loading'); } // 现在停止旧的视频流 if (this.currentVideoStream !== newStream) { const oldStream = this.currentVideoStream; setTimeout(() => { if (oldStream) { oldStream.getTracks().forEach(track => { track.stop(); this.logMessage(`已停止旧轨道: ${track.kind}`, 'info'); }); } }, 1000); // 延迟1秒停止旧流,确保新流已经稳定 } if (text) { this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`; this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success'); } else { this.currentVideoName.textContent = `视频流: ${videoFile}`; this.logMessage(`成功切换到视频流: ${videoFile}`, 'success'); } // 检查切换后的状态 setTimeout(() => { this.checkVideoStreamStatus(); }, 1000); } catch (error) { this.logMessage(`切换视频流失败: ${error.message}`, 'error'); this.recordedVideo.classList.remove('loading'); // 如果切换失败,尝试回到默认视频 if (videoFile !== this.defaultVideo) { this.logMessage('尝试回到默认视频', 'info'); await this.switchVideoStream(this.defaultVideo, 'fallback'); } } } bindEvents() { // 开始通话按钮 this.startButton.onclick = () => this.startCall(); // 停止通话按钮 this.stopButton.onclick = () => this.stopCall(); // 静音按钮 this.muteButton.onclick = () => this.toggleMute(); // 回到默认视频按钮 this.defaultVideoButton.onclick = () => this.returnToDefaultVideo(); // 测试视频文件按钮 this.testVideoButton.onclick = () => this.testAllVideoFiles(); // 发送文本按钮 this.sendTextButton.onclick = () => this.sendText(); // 回车键发送文本 this.textInput.onkeypress = (e) => { if (e.key === 'Enter') { this.sendText(); } }; // 语音输入按钮 this.startVoiceButton.onclick = () => this.startVoiceRecording(); this.stopVoiceButton.onclick = () => this.stopVoiceRecording(); } async startCall() { try { this.localStream = await navigator.mediaDevices.getUserMedia({ video: false, audio: true }); this.createPeerConnection(); this.startButton.disabled = true; this.stopButton.disabled = false; this.updateAudioStatus('已连接', 'connected'); this.logMessage('音频通话已开始', 'success'); // 确保视频映射已加载 if (Object.keys(this.videoMapping).length === 0) { await this.loadVideoMapping(); } this.logMessage(`视频映射已加载: ${Object.keys(this.videoMapping).length} 个映射`, 'info'); // 通知服务器通话开始 this.socket.emit('call-started'); } catch (error) { this.logMessage('无法访问麦克风: ' + error.message, 'error'); } } stopCall() { if (this.localStream) { this.localStream.getTracks().forEach(track => track.stop()); this.localStream = null; } if (this.peerConnection) { this.peerConnection.close(); this.peerConnection = null; } // 停止当前视频流 if (this.currentVideoStream) { this.currentVideoStream.getTracks().forEach(track => track.stop()); this.currentVideoStream = null; } this.recordedVideo.srcObject = null; this.currentVideo = null; this.currentVideoName.textContent = '未选择视频'; this.startButton.disabled = false; this.stopButton.disabled = true; this.updateAudioStatus('未连接', 'disconnected'); this.logMessage('音频通话已结束', 'info'); } createPeerConnection() { const configuration = { iceServers: [ { urls: "stun:stun.qq.com:3478" }, { urls: "stun:stun.miwifi.com:3478" }, { urls: 'stun:stun.l.google.com:19302' }, { urls: 'stun:stun1.l.google.com:19302' } ] }; this.peerConnection = new RTCPeerConnection(configuration); // 添加本地音频流 this.localStream.getTracks().forEach(track => { this.peerConnection.addTrack(track, this.localStream); }); // 处理远程流 this.peerConnection.ontrack = (event) => { this.remoteVideo.srcObject = event.streams[0]; }; // 处理ICE候选 this.peerConnection.onicecandidate = (event) => { if (event.candidate) { this.socket.emit('ice-candidate', { candidate: event.candidate }); } }; // 创建并发送offer this.peerConnection.createOffer() .then(offer => this.peerConnection.setLocalDescription(offer)) .then(() => { this.socket.emit('offer', { offer: this.peerConnection.localDescription }); }) .catch(error => { this.logMessage('创建offer失败: ' + error.message, 'error'); }); } async handleOffer(data) { if (!this.peerConnection) { await this.startCall(); } await this.peerConnection.setRemoteDescription(new RTCSessionDescription(data.offer)); const answer = await this.peerConnection.createAnswer(); await this.peerConnection.setLocalDescription(answer); this.socket.emit('answer', { answer: this.peerConnection.localDescription }); } async handleAnswer(data) { if (this.peerConnection) { await this.peerConnection.setRemoteDescription(new RTCSessionDescription(data.answer)); } } async handleIceCandidate(data) { if (this.peerConnection) { await this.peerConnection.addIceCandidate(new RTCIceCandidate(data.candidate)); } } toggleMute() { if (this.localStream) { const audioTrack = this.localStream.getAudioTracks()[0]; if (audioTrack) { audioTrack.enabled = !audioTrack.enabled; this.muteButton.textContent = audioTrack.enabled ? '静音' : '取消静音'; this.logMessage(audioTrack.enabled ? '已取消静音' : '已静音', 'info'); } } } async sendText() { const text = this.textInput.value.trim(); if (text) { this.socket.emit('text-input', { text }); this.logMessage(`发送文本: ${text}`, 'info'); this.textInput.value = ''; try { // 调用chat_with_audio进行大模型回答和音频合成 this.logMessage('正在处理文本,请稍候...', 'info'); const result = await chatWithAudioStream(text); this.logMessage(`大模型回答: ${result.llmResponse}`, 'success'); // 根据文本查找对应视频并切换 await this.handleTextInput(text); } catch (error) { this.logMessage(`处理文本失败: ${error.message}`, 'error'); console.error('chatWithAudioStream error:', error); } } } async handleTextInput(text) { // 根据文本查找对应视频 let videoFile = this.videoMapping['默认'] || this.defaultVideo; for (const [key, value] of Object.entries(this.videoMapping)) { if (text.toLowerCase().includes(key.toLowerCase())) { videoFile = value; break; } } // 切换到对应的视频流 await this.switchVideoStream(videoFile, 'text', text); // 通知服务器切换视频流 this.socket.emit('switch-video-stream', { videoFile, type: 'text', text }); } // 修改:使用音频处理器的语音录制功能 async startVoiceRecording() { const success = await this.audioProcessor.startRecording(); if (success) { this.startVoiceButton.disabled = true; this.stopVoiceButton.disabled = false; this.startVoiceButton.classList.add('recording'); this.voiceStatus.textContent = '等待语音输入...'; this.logMessage('高级语音录制已启动', 'success'); } else { this.voiceStatus.textContent = '录音启动失败'; } } // 修改:停止语音录制 stopVoiceRecording() { this.audioProcessor.stopRecording(); this.startVoiceButton.disabled = false; this.stopVoiceButton.disabled = true; this.startVoiceButton.classList.remove('recording'); this.voiceStatus.textContent = '点击开始语音输入'; this.logMessage('语音录制已停止', 'info'); } // 处理语音输入结果 async handleVoiceInput(text) { // 根据文本查找对应视频 let videoFile = this.videoMapping['默认'] || this.defaultVideo; for (const [key, value] of Object.entries(this.videoMapping)) { if (text.toLowerCase().includes(key.toLowerCase())) { videoFile = value; break; } } // 切换到对应的视频流 await this.switchVideoStream(videoFile, 'voice', text); // 通知服务器切换视频流 this.socket.emit('switch-video-stream', { videoFile, type: 'voice', text }); // 调用大模型处理 try { this.logMessage('正在处理语音输入,请稍候...', 'info'); const result = await chatWithAudioStream(text); this.logMessage(`大模型回答: ${result.llmResponse}`, 'success'); } catch (error) { this.logMessage(`处理语音输入失败: ${error.message}`, 'error'); console.error('chatWithAudioStream error:', error); } } // 删除原有的简单音频处理方法 // processVoiceInput() 和 simulateSpeechRecognition() 方法已被移除 simulateSpeechRecognition() { // 模拟语音识别,随机返回预设的文本 const texts = ['你好', '再见', '谢谢', 'hello', 'goodbye', 'thank you']; return texts[Math.floor(Math.random() * texts.length)]; } returnToDefaultVideo() { this.switchVideoStream(this.defaultVideo, 'default'); this.socket.emit('return-to-default'); this.logMessage(`已返回至默认视频: ${this.defaultVideo}`, 'info'); } updateStatus(message, type) { this.connectionStatus.textContent = message; this.connectionStatus.className = `status ${type}`; } updateAudioStatus(message, type) { this.audioStatus.textContent = message; this.audioStatus.className = `status-indicator ${type}`; } logMessage(message, type = 'info') { const logEntry = document.createElement('div'); logEntry.className = type; logEntry.textContent = `[${new Date().toLocaleTimeString()}] ${message}`; this.messageLog.appendChild(logEntry); this.messageLog.scrollTop = this.messageLog.scrollHeight; } checkVideoStreamStatus() { const status = { hasStream: !!this.currentVideoStream, streamTracks: this.currentVideoStream ? this.currentVideoStream.getTracks().length : 0, videoReadyState: this.recordedVideo.readyState, videoPaused: this.recordedVideo.paused, videoCurrentTime: this.recordedVideo.currentTime, videoDuration: this.recordedVideo.duration, currentVideo: this.currentVideo }; this.logMessage(`视频流状态: ${JSON.stringify(status)}`, 'info'); return status; } async testAllVideoFiles() { this.logMessage('开始测试所有视频文件...', 'info'); const videoFiles = ['asd.mp4', 'zxc.mp4', 'jkl.mp4']; for (const videoFile of videoFiles) { try { await this.testVideoFile(videoFile); } catch (error) { this.logMessage(`视频文件 ${videoFile} 测试失败: ${error.message}`, 'error'); } } this.logMessage('视频文件测试完成', 'info'); // 检查当前视频流状态 this.checkVideoStreamStatus(); } } // 页面加载完成后初始化应用 document.addEventListener('DOMContentLoaded', () => { new WebRTCChat(); });