// 用户输入文本后,进行大模型回答,并且合成音频,流式播放 import { requestLLMStream } from './llm_stream.js'; import { requestMinimaxi } from './minimaxi_stream.js'; import { getLLMConfig, getLLMConfigByScene, getMinimaxiConfig, getAudioConfig, validateConfig } from './config.js'; // 防止重复播放的标志 let isPlaying = false; // 音频播放队列 let audioQueue = []; let isProcessingQueue = false; // 历史消息缓存 let historyMessage = []; let isInitialized = false; // 初始化历史消息 async function initializeHistoryMessage(recentCount = 5) { if (isInitialized) return historyMessage; try { const response = await fetch(`/api/messages/for-llm?includeSystem=true&recentCount=${recentCount}`); if (!response.ok) { throw new Error('获取历史消息失败'); } const data = await response.json(); historyMessage = data.messages || []; isInitialized = true; console.log("历史消息初始化完成:", historyMessage.length, "条消息", historyMessage); return historyMessage; } catch (error) { console.error('获取历史消息失败,使用默认格式:', error); historyMessage = [ // { role: 'system', content: 'You are a helpful assistant.' } ]; isInitialized = true; return historyMessage; } } // 获取当前历史消息(同步方法) function getCurrentHistoryMessage() { if (!isInitialized) { console.warn('历史消息未初始化,返回默认消息'); return []; } return [...historyMessage]; // 返回副本,避免外部修改 } // 更新历史消息 function updateHistoryMessage(userInput, assistantResponse) { if (!isInitialized) { console.warn('历史消息未初始化,无法更新'); return; } historyMessage.push( { role: 'user', content: userInput }, { role: 'assistant', content: assistantResponse } ); // 可选:限制历史消息数量,保持最近的对话 const maxMessages = 20; // 保留最近10轮对话(20条消息) if (historyMessage.length > maxMessages) { // 保留系统消息和最近的对话 const systemMessages = historyMessage.filter(msg => msg.role === 'system'); const recentMessages = historyMessage.slice(-maxMessages + systemMessages.length); historyMessage = [...systemMessages, ...recentMessages.filter(msg => msg.role !== 'system')]; } } // 保存消息到服务端 async function saveMessage(userInput, assistantResponse) { try { // 验证参数是否有效 if (!userInput || !userInput.trim() || !assistantResponse || !assistantResponse.trim()) { console.warn('跳过保存消息:用户输入或助手回复为空'); return; } const response = await fetch('/api/messages/save', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ userInput: userInput.trim(), assistantResponse: assistantResponse.trim() }) }); if (!response.ok) { const errorData = await response.json().catch(() => ({})); throw new Error(`保存消息失败: ${response.status} ${errorData.error || response.statusText}`); } console.log('消息已保存到服务端'); } catch (error) { console.error('保存消息失败:', error); } } async function chatWithAudioStream(userInput) { // 确保历史消息已初始化 if (!isInitialized) { await initializeHistoryMessage(100); } // 验证配置 if (!validateConfig()) { throw new Error('配置不完整,请检查config.js文件中的API密钥设置'); } console.log('用户输入:', userInput); // 获取当前场景对应的配置 const llmConfig = await getLLMConfigByScene(); const minimaxiConfig = getMinimaxiConfig(); const audioConfig = getAudioConfig(); console.log(`当前场景: ${llmConfig.sceneName} (${llmConfig.sceneTag})`); console.log(`使用API Key: ${llmConfig.model}...`); // 清空音频队列 audioQueue = []; // 定义段落处理函数 const handleSegment = async (segment, textPlay) => { console.log('\n=== 处理文本段落 ==='); console.log('段落内容:', segment); try { // 为每个段落生成音频 const audioResult = await requestMinimaxi({ apiKey: minimaxiConfig.apiKey, groupId: minimaxiConfig.groupId, body: { model: audioConfig.model, text: segment, stream: audioConfig.stream, language_boost: audioConfig.language_boost, output_format: audioConfig.output_format, voice_setting: audioConfig.voiceSetting, audio_setting: audioConfig.audioSetting, }, stream: true, textPlay: textPlay }); // 将音频添加到播放队列 if (audioResult && audioResult.data && audioResult.data.audio) { audioQueue.push({ text: segment, audioHex: audioResult.data.audio }); console.log('音频已添加到队列,队列长度:', audioQueue.length); // this.socket.emit('audio-queue-updated', audioQueue.map(item => ({ text: item.text, hasAudio:!!item.audioHex }))); // 开始处理队列 // processAudioQueue(); } } catch (error) { console.error('生成音频失败:', error); } }; // 1. 获取包含历史的消息列表 console.log('\n=== 获取历史消息 ==='); const messages = getCurrentHistoryMessage(); messages.push({role: 'user', content: userInput}); console.log('发送的消息数量:', messages); // 2. 请求大模型回答 console.log('\n=== 请求大模型回答 ==='); const llmResponse = await requestLLMStream({ apiKey: llmConfig.apiKey, model: llmConfig.model, messages: messages, onSegment: handleSegment }); console.log('\n=== 大模型完整回答 ==='); console.log("llmResponse: ", llmResponse); // 3. 保存对话到服务端 await saveMessage(userInput, llmResponse); // 4. 更新本地历史消息 updateHistoryMessage(userInput, llmResponse); console.log('历史消息数量:', historyMessage.length); return { userInput, llmResponse, audioQueue: audioQueue.map(item => ({ text: item.text, hasAudio: !!item.audioHex })) }; } // 导出初始化函数,供外部调用 export { chatWithAudioStream, initializeHistoryMessage, getCurrentHistoryMessage, saveMessage, updateHistoryMessage }; // 处理音频播放队列 async function processAudioQueue() { if (isProcessingQueue) return; isProcessingQueue = true; // while (audioQueue.length > 0) { // const audioItem = audioQueue.shift(); // console.log('\n=== 播放队列中的音频 ==='); // console.log('文本:', audioItem.text); // try { // await playAudioStream(audioItem.audioHex); // } catch (error) { // console.error('播放音频失败:', error); // } // } isProcessingQueue = false; } // 流式播放音频 async function playAudioStream(audioHex) { console.log('=== 开始播放音频 ==='); console.log('音频数据长度:', audioHex.length); // 将hex转换为ArrayBuffer const audioBuffer = hexToArrayBuffer(audioHex); // 创建AudioContext const audioContext = new (window.AudioContext || window.webkitAudioContext)(); try { // 解码音频 const audioData = await audioContext.decodeAudioData(audioBuffer); // 创建音频源 const source = audioContext.createBufferSource(); source.buffer = audioData; source.connect(audioContext.destination); // 播放 source.start(0); console.log('音频播放开始,时长:', audioData.duration, '秒'); // 等待播放完成 return new Promise((resolve) => { source.onended = () => { console.log('音频播放完成'); resolve(); }; }); } catch (error) { console.error('音频播放失败:', error); throw error; } } // 将hex字符串转换为ArrayBuffer function hexToArrayBuffer(hex) { const bytes = new Uint8Array(hex.length / 2); for (let i = 0; i < hex.length; i += 2) { bytes[i / 2] = parseInt(hex.substr(i, 2), 16); } return bytes.buffer; } // 在Node.js环境下的音频播放(使用play-sound库) async function playAudioStreamNode(audioHex) { // 检查是否在Node.js环境中 if (typeof window !== 'undefined') { console.warn('playAudioStreamNode 只能在Node.js环境中使用'); return; } try { const fs = require('fs'); const path = require('path'); // 将hex转换为buffer const audioBuffer = Buffer.from(audioHex, 'hex'); // 保存为临时文件 const tempFile = path.join(process.cwd(), 'temp_audio.mp3'); fs.writeFileSync(tempFile, audioBuffer); // 使用系统默认播放器播放 const { exec } = require('child_process'); const platform = process.platform; let command; if (platform === 'win32') { command = `start "" "${tempFile}"`; } else if (platform === 'darwin') { command = `open "${tempFile}"`; } else { command = `xdg-open "${tempFile}"`; } exec(command, (error) => { if (error) { console.error('播放音频失败:', error); } else { console.log('音频播放开始'); } }); // 等待一段时间后删除临时文件 setTimeout(() => { if (fs.existsSync(tempFile)) { fs.unlinkSync(tempFile); } }, 10000); } catch (error) { console.error('音频播放失败:', error); throw error; } } // export { chatWithAudioStream, playAudioStream, playAudioStreamNode, initializeHistoryMessage, getCurrentHistoryMessage };