313 lines
9.6 KiB
JavaScript
313 lines
9.6 KiB
JavaScript
// 用户输入文本后，进行大模型回答，并且合成音频，流式播放
|
||
|
||
import { requestLLMStream } from './llm_stream.js';
|
||
import { requestMinimaxi } from './minimaxi_stream.js';
|
||
import { getLLMConfig, getMinimaxiConfig, getAudioConfig, validateConfig } from './config.js';
|
||
|
||
// 防止重复播放的标志
|
||
let isPlaying = false;
|
||
// 音频播放队列
|
||
let audioQueue = [];
|
||
let isProcessingQueue = false;
|
||
|
||
// 历史消息缓存
|
||
let historyMessage = [];
|
||
let isInitialized = false;
|
||
|
||
// 初始化历史消息
|
||
async function initializeHistoryMessage(recentCount = 5) {
|
||
if (isInitialized) return historyMessage;
|
||
|
||
try {
|
||
const response = await fetch(`/api/messages/for-llm?includeSystem=true&recentCount=${recentCount}`);
|
||
if (!response.ok) {
|
||
throw new Error('获取历史消息失败');
|
||
}
|
||
const data = await response.json();
|
||
historyMessage = data.messages || [];
|
||
isInitialized = true;
|
||
console.log("历史消息初始化完成:", historyMessage.length, "条消息");
|
||
return historyMessage;
|
||
} catch (error) {
|
||
console.error('获取历史消息失败，使用默认格式:', error);
|
||
historyMessage = [
|
||
{ role: 'system', content: 'You are a helpful assistant.' }
|
||
];
|
||
isInitialized = true;
|
||
return historyMessage;
|
||
}
|
||
}
|
||
|
||
// 获取当前历史消息（同步方法）
|
||
function getCurrentHistoryMessage() {
|
||
if (!isInitialized) {
|
||
console.warn('历史消息未初始化，返回默认消息');
|
||
return [{ role: 'system', content: 'You are a helpful assistant.' }];
|
||
}
|
||
return [...historyMessage]; // 返回副本，避免外部修改
|
||
}
|
||
|
||
// 更新历史消息
|
||
function updateHistoryMessage(userInput, assistantResponse) {
|
||
if (!isInitialized) {
|
||
console.warn('历史消息未初始化，无法更新');
|
||
return;
|
||
}
|
||
|
||
historyMessage.push(
|
||
{ role: 'user', content: userInput },
|
||
{ role: 'assistant', content: assistantResponse }
|
||
);
|
||
|
||
// 可选：限制历史消息数量，保持最近的对话
|
||
const maxMessages = 20; // 保留最近10轮对话（20条消息）
|
||
if (historyMessage.length > maxMessages) {
|
||
// 保留系统消息和最近的对话
|
||
const systemMessages = historyMessage.filter(msg => msg.role === 'system');
|
||
const recentMessages = historyMessage.slice(-maxMessages + systemMessages.length);
|
||
historyMessage = [...systemMessages, ...recentMessages.filter(msg => msg.role !== 'system')];
|
||
}
|
||
}
|
||
|
||
// 保存消息到服务端
|
||
async function saveMessage(userInput, assistantResponse) {
|
||
try {
|
||
const response = await fetch('/api/messages/save', {
|
||
method: 'POST',
|
||
headers: {
|
||
'Content-Type': 'application/json'
|
||
},
|
||
body: JSON.stringify({
|
||
userInput,
|
||
assistantResponse
|
||
})
|
||
});
|
||
|
||
if (!response.ok) {
|
||
throw new Error('保存消息失败');
|
||
}
|
||
|
||
console.log('消息已保存到服务端');
|
||
} catch (error) {
|
||
console.error('保存消息失败:', error);
|
||
}
|
||
}
|
||
|
||
async function chatWithAudioStream(userInput) {
|
||
// 确保历史消息已初始化
|
||
if (!isInitialized) {
|
||
await initializeHistoryMessage();
|
||
}
|
||
|
||
// 验证配置
|
||
if (!validateConfig()) {
|
||
throw new Error('配置不完整，请检查config.js文件中的API密钥设置');
|
||
}
|
||
|
||
console.log('用户输入:', userInput);
|
||
|
||
// 获取配置
|
||
const llmConfig = getLLMConfig();
|
||
const minimaxiConfig = getMinimaxiConfig();
|
||
const audioConfig = getAudioConfig();
|
||
|
||
// 清空音频队列
|
||
audioQueue = [];
|
||
|
||
// 定义段落处理函数
|
||
const handleSegment = async (segment) => {
|
||
console.log('\n=== 处理文本段落 ===');
|
||
console.log('段落内容:', segment);
|
||
|
||
try {
|
||
// 为每个段落生成音频
|
||
const audioResult = await requestMinimaxi({
|
||
apiKey: minimaxiConfig.apiKey,
|
||
groupId: minimaxiConfig.groupId,
|
||
body: {
|
||
model: audioConfig.model,
|
||
text: segment,
|
||
stream: audioConfig.stream,
|
||
language_boost: audioConfig.language_boost,
|
||
output_format: audioConfig.output_format,
|
||
voice_setting: audioConfig.voiceSetting,
|
||
audio_setting: audioConfig.audioSetting,
|
||
},
|
||
stream: true,
|
||
});
|
||
|
||
// 将音频添加到播放队列
|
||
if (audioResult && audioResult.data && audioResult.data.audio) {
|
||
audioQueue.push({
|
||
text: segment,
|
||
audioHex: audioResult.data.audio
|
||
});
|
||
console.log('音频已添加到队列，队列长度:', audioQueue.length);
|
||
// this.socket.emit('audio-queue-updated', audioQueue.map(item => ({ text: item.text, hasAudio:!!item.audioHex })));
|
||
// 开始处理队列
|
||
// processAudioQueue();
|
||
}
|
||
} catch (error) {
|
||
console.error('生成音频失败:', error);
|
||
}
|
||
};
|
||
|
||
// 1. 获取包含历史的消息列表
|
||
console.log('\n=== 获取历史消息 ===');
|
||
const messages = getCurrentHistoryMessage();
|
||
messages.push({role: 'user', content: userInput});
|
||
console.log('发送的消息数量:', messages);
|
||
|
||
// 2. 请求大模型回答
|
||
console.log('\n=== 请求大模型回答 ===');
|
||
const llmResponse = await requestLLMStream({
|
||
apiKey: llmConfig.apiKey,
|
||
model: llmConfig.model,
|
||
messages: messages,
|
||
onSegment: handleSegment
|
||
});
|
||
|
||
console.log('\n=== 大模型完整回答 ===');
|
||
console.log("llmResponse: ", llmResponse);
|
||
|
||
// 3. 保存对话到服务端
|
||
await saveMessage(userInput, llmResponse);
|
||
|
||
// 4. 更新本地历史消息
|
||
updateHistoryMessage(userInput, llmResponse);
|
||
console.log('历史消息数量:', historyMessage.length);
|
||
|
||
return {
|
||
userInput,
|
||
llmResponse,
|
||
audioQueue: audioQueue.map(item => ({ text: item.text, hasAudio: !!item.audioHex }))
|
||
};
|
||
}
|
||
|
||
// 导出初始化函数，供外部调用
|
||
export { chatWithAudioStream, initializeHistoryMessage, getCurrentHistoryMessage };
|
||
|
||
// 处理音频播放队列
|
||
async function processAudioQueue() {
|
||
if (isProcessingQueue) return;
|
||
|
||
isProcessingQueue = true;
|
||
|
||
// while (audioQueue.length > 0) {
|
||
// const audioItem = audioQueue.shift();
|
||
// console.log('\n=== 播放队列中的音频 ===');
|
||
// console.log('文本:', audioItem.text);
|
||
|
||
// try {
|
||
// await playAudioStream(audioItem.audioHex);
|
||
// } catch (error) {
|
||
// console.error('播放音频失败:', error);
|
||
// }
|
||
// }
|
||
|
||
isProcessingQueue = false;
|
||
}
|
||
|
||
// 流式播放音频
|
||
async function playAudioStream(audioHex) {
|
||
console.log('=== 开始播放音频 ===');
|
||
console.log('音频数据长度:', audioHex.length);
|
||
|
||
// 将hex转换为ArrayBuffer
|
||
const audioBuffer = hexToArrayBuffer(audioHex);
|
||
|
||
// 创建AudioContext
|
||
const audioContext = new (window.AudioContext || window.webkitAudioContext)();
|
||
|
||
try {
|
||
// 解码音频
|
||
const audioData = await audioContext.decodeAudioData(audioBuffer);
|
||
|
||
// 创建音频源
|
||
const source = audioContext.createBufferSource();
|
||
source.buffer = audioData;
|
||
source.connect(audioContext.destination);
|
||
|
||
// 播放
|
||
source.start(0);
|
||
|
||
console.log('音频播放开始，时长:', audioData.duration, '秒');
|
||
|
||
// 等待播放完成
|
||
return new Promise((resolve) => {
|
||
source.onended = () => {
|
||
console.log('音频播放完成');
|
||
resolve();
|
||
};
|
||
});
|
||
} catch (error) {
|
||
console.error('音频播放失败:', error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
// 将hex字符串转换为ArrayBuffer
|
||
function hexToArrayBuffer(hex) {
|
||
const bytes = new Uint8Array(hex.length / 2);
|
||
for (let i = 0; i < hex.length; i += 2) {
|
||
bytes[i / 2] = parseInt(hex.substr(i, 2), 16);
|
||
}
|
||
return bytes.buffer;
|
||
}
|
||
|
||
// 在Node.js环境下的音频播放（使用play-sound库）
|
||
async function playAudioStreamNode(audioHex) {
|
||
// 检查是否在Node.js环境中
|
||
if (typeof window !== 'undefined') {
|
||
console.warn('playAudioStreamNode 只能在Node.js环境中使用');
|
||
return;
|
||
}
|
||
|
||
try {
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
|
||
// 将hex转换为buffer
|
||
const audioBuffer = Buffer.from(audioHex, 'hex');
|
||
|
||
// 保存为临时文件
|
||
const tempFile = path.join(process.cwd(), 'temp_audio.mp3');
|
||
fs.writeFileSync(tempFile, audioBuffer);
|
||
|
||
// 使用系统默认播放器播放
|
||
const { exec } = require('child_process');
|
||
const platform = process.platform;
|
||
|
||
let command;
|
||
if (platform === 'win32') {
|
||
command = `start "" "${tempFile}"`;
|
||
} else if (platform === 'darwin') {
|
||
command = `open "${tempFile}"`;
|
||
} else {
|
||
command = `xdg-open "${tempFile}"`;
|
||
}
|
||
|
||
exec(command, (error) => {
|
||
if (error) {
|
||
console.error('播放音频失败:', error);
|
||
} else {
|
||
console.log('音频播放开始');
|
||
}
|
||
});
|
||
|
||
// 等待一段时间后删除临时文件
|
||
setTimeout(() => {
|
||
if (fs.existsSync(tempFile)) {
|
||
fs.unlinkSync(tempFile);
|
||
}
|
||
}, 10000);
|
||
|
||
} catch (error) {
|
||
console.error('音频播放失败:', error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
|
||
|
||
// export { chatWithAudioStream, playAudioStream, playAudioStreamNode, initializeHistoryMessage, getCurrentHistoryMessage };
|