diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..b4032c8 --- /dev/null +++ b/TODO.md @@ -0,0 +1,3 @@ +- ❌ Auto mute XiaoAi reply +- Stream response +- Update long/short memories diff --git a/package.json b/package.json index cc6acc5..df14987 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mi-gpt", - "version": "0.1.0", + "version": "0.2.0", "type": "module", "description": "Seamlessly integrate your XiaoAI speaker and Mi Home devices with ChatGPT for an enhanced smart home experience.", "license": "MIT", @@ -27,6 +27,7 @@ "dependencies": { "@prisma/client": "^5.8.1", "axios": "^1.6.5", + "mi-service-lite": "^2.0.0", "openai": "^4.25.0", "prisma": "^5.8.1" }, diff --git a/src/services/speaker/ai.ts b/src/services/speaker/ai.ts new file mode 100644 index 0000000..ec4efc1 --- /dev/null +++ b/src/services/speaker/ai.ts @@ -0,0 +1,220 @@ +import { pickOne } from "../../utils/base"; +import { + Speaker, + SpeakerCommand, + SpeakerConfig, + QueryMessage, +} from "./speaker"; + +export type AISpeakerConfig = SpeakerConfig & { + askAI?: (msg: QueryMessage) => Promise; + /** + * 切换音色前缀 + * + * 比如:音色切换到(文静毛毛) + */ + switchSpeakerPrefix?: string; + /** + * AI 开始回答时的提示语 + * + * 比如:请稍等,让我想想 + */ + onAIAsking?: string[]; + /** + * AI 回答异常时的提示语 + * + * 比如:出错了,请稍后再试吧! + */ + onAIError?: string[]; + /** + * 设备名称,用来唤醒/退出对话模式等 + * + * 建议使用常见词语,避免使用多音字和容易混淆读音的词语 + */ + name?: string; + /** + * 召唤关键词 + * + * 当消息中包含召唤关键词时,会调用 AI 来响应用户消息 + * + * 比如:打开/进入/召唤豆包 + */ + callAIPrefix?: string[]; + /** + * 唤醒关键词 + * + * 当消息中包含唤醒关键词时,会进入 AI 唤醒状态 + * + * 比如:关闭/退出/再见豆包 + */ + wakeUpKeyWords?: string[]; + /** + * 退出关键词 + * + * 当消息中包含退出关键词时,会退出 AI 唤醒状态 + */ + exitKeywords?: string[]; + /** + * 进入 AI 模式的欢迎语 + * + * 比如:你好,我是豆包,请问有什么能够帮你的吗? + */ + onEnterAI?: string[]; + /** + * 退出 AI 模式的提示语 + * + * 比如:豆包已退出 + */ + onExitAI?: string[]; +}; + +type AnswerStep = ( + msg: any, + data: any +) => Promise<{ stop?: boolean; data?: any } | void>; + +export class AISpeaker extends Speaker { + askAI: AISpeakerConfig["askAI"]; + name: string; + switchSpeakerPrefix: string; + onEnterAI: string[]; + onExitAI: string[]; + callAIPrefix: string[]; + wakeUpKeyWords: string[]; + exitKeywords: string[]; + onAIAsking: string[]; + onAIError: string[]; + + constructor(config: AISpeakerConfig) { + super(config); + const { + askAI, + name = "豆包", + switchSpeakerPrefix = "音色切换到", + wakeUpKeyWords = ["打开", "进入", "召唤"], + exitKeywords = ["关闭", "退出", "再见"], + onAIAsking = ["让我先想想", "请稍等"], + onAIError = ["啊哦,出错了,请稍后再试吧!"], + } = config; + this.askAI = askAI; + this.switchSpeakerPrefix = switchSpeakerPrefix; + this.name = name; + this.onAIError = onAIError; + this.onAIAsking = onAIAsking; + this.wakeUpKeyWords = wakeUpKeyWords.map((e) => e + this.name); + this.exitKeywords = exitKeywords.map((e) => e + this.name); + this.onEnterAI = config.onEnterAI ?? [ + `你好,我是${this.name},很高兴为你服务!`, + ]; + this.onExitAI = config.onExitAI ?? [`${this.name}已关闭!`]; + this.callAIPrefix = config.callAIPrefix ?? [ + "请", + "你", + this.name, + "问问" + this.name, + ]; + } + + async enterKeepAlive() { + // 回应 + await this.response({ text: pickOne(this.onEnterAI)!, keepAlive: true }); + // 唤醒 + await super.enterKeepAlive(); + } + + async exitKeepAlive() { + // 退出唤醒状态 + await super.exitKeepAlive(); + // 回应 + await this.response({ + text: pickOne(this.onExitAI)!, + keepAlive: false, + playSFX: false, + }); + await this.unWakeUp(); + } + + get commands() { + return [ + { + match: (msg) => this.wakeUpKeyWords.some((e) => msg.text.includes(e)), + run: async (msg) => { + await this.enterKeepAlive(); + }, + }, + { + match: (msg) => this.exitKeywords.some((e) => msg.text.includes(e)), + run: async (msg) => { + await this.exitKeepAlive(); + }, + }, + { + match: (msg) => msg.text.startsWith(this.switchSpeakerPrefix), + run: async (msg) => { + await this.response({ + text: "正在切换音色,请稍等...", + keepAlive: this.keepAlive, + }); + const speaker = msg.text.replace(this.switchSpeakerPrefix, ""); + const success = await this.switchDefaultSpeaker(speaker); + await this.response({ + text: success ? "音色已切换!" : "音色切换失败!", + keepAlive: this.keepAlive, + }); + }, + }, + ...this._commands, + { + match: (msg) => + this.keepAlive || + this.callAIPrefix.some((e) => msg.text.startsWith(e)), + run: (msg) => this.askAIForAnswer(msg), + }, + ] as SpeakerCommand[]; + } + + private _askAIForAnswerSteps: AnswerStep[] = [ + async (msg, data) => { + // 思考中 + await this.response({ + audio: process.env.AUDIO_ACTIVE, + text: pickOne(this.onAIAsking)!, + keepAlive: this.keepAlive, + }); + }, + async (msg, data) => { + // 调用 AI 获取回复 + let answer = await this.askAI?.(msg); + return { data: { answer } }; + }, + async (msg, data) => { + if (!data.answer) { + // 回答异常 + await this.response({ + audio: process.env.AUDIO_ERROR, + text: pickOne(this.onAIError)!, + keepAlive: this.keepAlive, + }); + } + }, + ]; + + async askAIForAnswer(msg: QueryMessage) { + let data: any = {}; + const { hasNewMsg } = this.checkIfHasNewMsg(msg); + for (const action of this._askAIForAnswerSteps) { + const res = await action(msg, data); + if (hasNewMsg()) { + // 收到新的用户请求消息,终止后续操作和响应 + return; + } + if (res?.data) { + data = { ...data, ...res.data }; + } + if (res?.stop) { + break; + } + } + return data.answer; + } +} diff --git a/src/services/speaker/base.ts b/src/services/speaker/base.ts new file mode 100644 index 0000000..22e59af --- /dev/null +++ b/src/services/speaker/base.ts @@ -0,0 +1,158 @@ +import { assert } from "console"; +import { + MiServiceConfig, + getMiIOT, + getMiNA, + MiNA, + MiIOT, +} from "mi-service-lite"; +import { sleep } from "../../utils/base"; +import { Http } from "../http"; + +export type TTSProvider = "xiaoai" | "doubao"; + +type Speaker = { + name: string; + gender: "男" | "女"; + speaker: string; +}; + +export type BaseSpeakerConfig = MiServiceConfig & { + // 语音合成服务商 + tts?: TTSProvider; + // 检测间隔(单位毫秒,默认 100 毫秒) + interval?: number; +}; + +export class BaseSpeaker { + MiNA?: MiNA; + MiIOT?: MiIOT; + + interval: number; + tts: TTSProvider; + config: MiServiceConfig; + constructor(config: BaseSpeakerConfig) { + this.config = config; + const { interval = 100, tts = "doubao" } = config; + this.interval = interval; + this.tts = tts; + } + + async initMiServices() { + this.MiNA = await getMiNA(this.config); + this.MiIOT = await getMiIOT(this.config); + assert(!!this.MiNA && !!this.MiIOT, "❌ init Mi Services failed"); + } + + wakeUp() { + return this.MiIOT!.doAction(5, 3); + } + + async unWakeUp() { + await this.MiIOT!.setProperty(4, 1, true); // 关闭麦克风 + await this.MiIOT!.setProperty(4, 1, false); // 打开麦克风 + } + + responding = false; + async response(options: { + tts?: TTSProvider; + text?: string; + audio?: string; + speaker?: string; + keepAlive?: boolean; + playSFX?: boolean; + }) { + let { + text, + audio, + playSFX = true, + keepAlive = false, + tts = this.tts, + speaker = this._defaultSpeaker, + } = options ?? {}; + + // 播放回复 + const play = async (args?: { tts?: string; url?: string }) => { + const ttsNotXiaoai = !audio && tts !== "xiaoai"; + playSFX = ttsNotXiaoai && playSFX; + // 播放开始提示音 + if (playSFX) { + await this.MiNA!.play({ url: process.env.AUDIO_BEEP }); + } + // 在播放 TTS 语音之前,先取消小爱音箱的唤醒状态,防止将 TTS 语音识别成用户指令 + if (ttsNotXiaoai) { + await this.unWakeUp(); + } + await this.MiNA!.play(args); + console.log("✅ " + text ?? audio); + // 等待回答播放完毕 + while (true) { + const res = await this.MiNA!.getStatus(); + if ( + !this.responding || // 有新消息 + (res?.status === "playing" && res?.media_type) // 小爱自己开始播放音乐 + ) { + // 响应被中断 + return "break"; + } + if (res?.status && res.status !== "playing") { + break; + } + await sleep(this.interval); + } + // 播放结束提示音 + if (playSFX) { + await this.MiNA!.play({ url: process.env.AUDIO_BEEP }); + } + // 保持唤醒状态 + if (keepAlive) { + await this.wakeUp(); + } + }; + + // 开始响应 + let res; + this.responding = true; + if (audio) { + // 音频回复 + res = await play({ url: audio }); + } else if (text) { + // 文字回复 + switch (tts) { + case "doubao": + text = encodeURIComponent(text); + const doubaoTTS = process.env.TTS_DOUBAO; + const url = `${doubaoTTS}?speaker=${speaker}&text=${text}`; + res = await play({ url }); + break; + case "xiaoai": + default: + res = await play({ tts: text }); + } + this.responding = false; + return res; + } + } + + private _doubaoSpeakers?: Speaker[]; + private _defaultSpeaker = "zh_female_maomao_conversation_wvae_bigtts"; + async switchDefaultSpeaker(speaker: string) { + if (!this._doubaoSpeakers) { + const doubaoSpeakers = process.env.SPEAKERS_DOUBAO; + const res = await Http.get(doubaoSpeakers ?? "/"); + if (Array.isArray(res)) { + this._doubaoSpeakers = res; + } + } + if (!this._doubaoSpeakers) { + return false; + } + const target = this._doubaoSpeakers.find( + (e) => e.name === speaker || e.speaker === speaker + ); + if (target) { + this._defaultSpeaker = target.speaker; + } + return this._defaultSpeaker === target?.speaker; + } +} diff --git a/src/services/speaker/speaker.ts b/src/services/speaker/speaker.ts new file mode 100644 index 0000000..02512ff --- /dev/null +++ b/src/services/speaker/speaker.ts @@ -0,0 +1,274 @@ +import { firstOf, lastOf, sleep } from "../../utils/base"; +import { BaseSpeaker, BaseSpeakerConfig } from "./base"; + +export interface QueryMessage { + text: string; + answer: string; + /** + * 毫秒 + */ + timestamp: number; +} + +export interface SpeakerCommand { + match: (msg: QueryMessage) => boolean; + /** + * 命中后执行的操作,返回值非空时会自动回复给用户 + */ + run: (msg: QueryMessage) => Promise; +} + +export type SpeakerConfig = BaseSpeakerConfig & { + /** + * 拉取消息心跳间隔(单位毫秒,默认1秒) + */ + heartbeat?: number; + /** + * 自定义的消息指令 + */ + commands?: SpeakerCommand[]; + /** + * 无响应一段时间后,多久自动退出唤醒模式(单位秒,默认30秒) + */ + exitKeepAliveAfter?: number; +}; + +export class Speaker extends BaseSpeaker { + heartbeat: number; + exitKeepAliveAfter: number; + currentQueryMsg?: QueryMessage; + + constructor(config: SpeakerConfig) { + super(config); + const { heartbeat = 1000, exitKeepAliveAfter = 30 } = config; + this._commands = config.commands ?? []; + this.heartbeat = heartbeat; + this.exitKeepAliveAfter = exitKeepAliveAfter; + } + + private _status: "running" | "stopped" = "running"; + + stop() { + this._status = "stopped"; + } + + async run() { + await this.initMiServices(); + if (!this.MiNA) { + this.stop(); + } + console.log("✅ 服务已启动..."); + this.activeKeepAliveMode(); + while (this._status === "running") { + const nextMsg = await this.fetchNextMessage(); + if (nextMsg) { + this.responding = false; + console.log("🔥 " + nextMsg.text); + // 异步处理消息,不阻塞正常消息拉取 + this.onMessage(nextMsg); + } + await sleep(this.heartbeat); + } + } + + async activeKeepAliveMode() { + while (this._status === "running") { + if (this.keepAlive) { + // 唤醒中 + if (!this.responding) { + // 没有回复时,一直播放静音音频使小爱闭嘴 + await this.MiNA?.play({ url: process.env.AUDIO_SILENT }); + } + } + await sleep(this.interval); + } + } + + _commands: SpeakerCommand[] = []; + get commands() { + return this._commands; + } + + addCommand(command: SpeakerCommand) { + this.commands.push(command); + } + + async onMessage(msg: QueryMessage) { + const { noNewMsg } = this.checkIfHasNewMsg(msg); + for (const command of this.commands) { + if (command.match(msg)) { + // 关闭小爱的回复 + await this.MiNA!.pause(); + // 执行命令 + const answer = await command.run(msg); + // 回复用户 + if (answer) { + if (noNewMsg()) { + await this.response({ + text: answer, + keepAlive: this.keepAlive, + }); + } + } + await this.exitKeepAliveIfNeeded(); + return; + } + } + } + + /** + * 是否保持设备响应状态 + */ + keepAlive = false; + + async enterKeepAlive() { + // 唤醒 + this.keepAlive = true; + } + + async exitKeepAlive() { + // 退出唤醒状态 + this.keepAlive = false; + } + + private _preTimer: any; + async exitKeepAliveIfNeeded() { + // 无响应一段时间后自动退出唤醒状态 + if (this._preTimer) { + clearTimeout(this._preTimer); + } + const { noNewMsg } = this.checkIfHasNewMsg(); + this._preTimer = setTimeout(async () => { + if (this.keepAlive && !this.responding && noNewMsg()) { + await this.exitKeepAlive(); + } + }, this.exitKeepAliveAfter * 1000); + } + + checkIfHasNewMsg(currentMsg?: QueryMessage) { + const currentTimestamp = (currentMsg ?? this.currentQueryMsg)?.timestamp; + return { + hasNewMsg: () => currentTimestamp !== this.currentQueryMsg?.timestamp, + noNewMsg: () => currentTimestamp === this.currentQueryMsg?.timestamp, + }; + } + + private _tempMsgs: QueryMessage[] = []; + async fetchNextMessage(): Promise { + if (!this.currentQueryMsg) { + await this._fetchFirstMessage(); + // 第一条消息仅用作初始化消息游标,不响应 + return; + } + return this._fetchNextMessage(); + } + + private async _fetchFirstMessage() { + const msgs = await this.getMessages({ + limit: 1, + filterTTS: false, + }); + this.currentQueryMsg = msgs[0]; + } + + private async _fetchNextMessage(): Promise { + if (this._tempMsgs.length > 0) { + // 当前有暂存的新消息(从新到旧),依次处理之 + return this._fetchNextTempMessage(); + } + // 拉取最新的 2 条 msg(用于和上一条消息比对是否连续) + const nextMsg = await this._fetchNext2Messages(); + if (nextMsg !== "continue") { + return nextMsg; + } + // 继续向上拉取其他新消息 + return this._fetchNextRemainingMessages(); + } + + private async _fetchNext2Messages() { + // 拉取最新的 2 条 msg(用于和上一条消息比对是否连续) + let msgs = await this.getMessages({ limit: 2 }); + if ( + msgs.length < 1 || + firstOf(msgs)!.timestamp <= this.currentQueryMsg!.timestamp + ) { + // 没有拉到新消息 + return; + } + if ( + firstOf(msgs)!.timestamp > this.currentQueryMsg!.timestamp && + (msgs.length === 1 || + lastOf(msgs)!.timestamp <= this.currentQueryMsg!.timestamp) + ) { + // 刚好收到一条新消息 + this.currentQueryMsg = firstOf(msgs); + return this.currentQueryMsg; + } + // 还有其他新消息,暂存当前的新消息 + for (const msg of msgs) { + if (msg.timestamp > this.currentQueryMsg!.timestamp) { + this._tempMsgs.push(msg); + } + } + return "continue"; + } + + private _fetchNextTempMessage() { + const nextMsg = this._tempMsgs.pop(); + this.currentQueryMsg = nextMsg; + return nextMsg; + } + + private async _fetchNextRemainingMessages(maxPage = 3) { + // 继续向上拉取其他新消息 + let currentPage = 0; + while (true) { + currentPage++; + if (currentPage > maxPage) { + // 拉取新消息超长,取消拉取 + return this._fetchNextTempMessage(); + } + const nextTimestamp = lastOf(this._tempMsgs)!.timestamp; + const msgs = await this.getMessages({ + limit: 10, + timestamp: nextTimestamp, + }); + for (const msg of msgs) { + if (msg.timestamp >= nextTimestamp) { + // 忽略上一页的消息 + continue; + } else if (msg.timestamp > this.currentQueryMsg!.timestamp) { + // 继续添加新消息 + this._tempMsgs.push(msg); + } else { + // 拉取到历史消息处 + return this._fetchNextTempMessage(); + } + } + } + } + + async getMessages(options?: { + limit?: number; + timestamp?: number; + filterTTS?: boolean; + }): Promise { + const filterTTS = options?.filterTTS ?? true; + const conversation = await this.MiNA!.getConversations(options); + let records = conversation?.records ?? []; + if (filterTTS) { + // 过滤有小爱回答的消息 + records = records.filter( + (e) => e.answers.length > 0 && e.answers.some((e) => e.type === "TTS") + ); + } + return records.map((e) => { + const ttsAnswer = e.answers.find((e) => e.type === "TTS") as any; + return { + text: e.query, + answer: ttsAnswer?.tts?.text, + timestamp: e.time, + }; + }); + } +} diff --git a/src/utils/base.ts b/src/utils/base.ts index e810295..e98b41a 100644 --- a/src/utils/base.ts +++ b/src/utils/base.ts @@ -4,7 +4,7 @@ export function timestamp() { return new Date().getTime(); } -export async function delay(time: number) { +export async function sleep(time: number) { return new Promise((resolve) => setTimeout(resolve, time)); } diff --git a/tests/db/index.ts b/tests/db.ts similarity index 89% rename from tests/db/index.ts rename to tests/db.ts index ed0c4ae..09adb2b 100644 --- a/tests/db/index.ts +++ b/tests/db.ts @@ -1,7 +1,7 @@ import { assert } from "console"; -import { ConversationManager } from "../../src/services/bot/conversation"; -import { println } from "../../src/utils/base"; -import { MessageCRUD } from "../../src/services/db/message"; +import { ConversationManager } from "../src/services/bot/conversation"; +import { println } from "../src/utils/base"; +import { MessageCRUD } from "../src/services/db/message"; export async function testDB() { const manager = new ConversationManager({ diff --git a/tests/speaker.ts b/tests/speaker.ts new file mode 100644 index 0000000..580f361 --- /dev/null +++ b/tests/speaker.ts @@ -0,0 +1,58 @@ +import { AISpeaker } from "../src/services/speaker/ai"; +import { sleep } from "../src/utils/base"; + +export async function main() { + const config: any = { + userId: process.env.MI_USER!, + password: process.env.MI_PASS!, + did: process.env.MI_DID, + tts: "doubao", + }; + + const speaker = new AISpeaker(config); + await speaker.initMiServices(); + // await testSpeakerResponse(speaker); + // await testSpeakerGetMessages(speaker); + // await testSwitchSpeaker(speaker); + // await testSpeakerUnWakeUp(speaker); + await testAISpeaker(speaker); +} + +async function testAISpeaker(speaker: AISpeaker) { + speaker.askAI = async (msg) => { + return "你说:" + msg.text; + }; + await speaker.run(); + console.log("finished"); +} + +async function testSpeakerUnWakeUp(speaker: AISpeaker) { + await speaker.wakeUp(); + await sleep(1000); + await speaker.unWakeUp(); + console.log("hello"); +} + +async function testSwitchSpeaker(speaker: AISpeaker) { + await speaker.response({ text: "你好,我是豆包,很高兴认识你!" }); + const success = await speaker.switchDefaultSpeaker("魅力苏菲"); + console.log("switchDefaultSpeaker 魅力苏菲", success); + await speaker.response({ text: "你好,我是豆包,很高兴认识你!" }); + console.log("hello"); +} + +async function testSpeakerGetMessages(speaker: AISpeaker) { + let msgs = await speaker.getMessages({ filterTTS: true }); + console.log("filterTTS msgs", msgs); + msgs = await speaker.getMessages({ filterTTS: false }); + console.log("no filterTTS msgs", msgs); +} + +async function testSpeakerResponse(speaker: AISpeaker) { + let status = await speaker.MiNA!.getStatus(); + console.log("curent status", status); + speaker.response({ text: "你好,我是豆包,很高兴认识你!" }); + sleep(1000); + status = await speaker.MiNA!.getStatus(); + console.log("tts status", status); +} diff --git a/tsup.config.ts b/tsup.config.ts index 03699e7..12d12d8 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -6,7 +6,6 @@ export default defineConfig(() => ({ target: "node16", platform: "node", format: ["esm", "cjs"], - splitting: false, sourcemap: false, treeshake: true, minify: true, diff --git a/yarn.lock b/yarn.lock index 652a012..c77e5fa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -874,6 +874,12 @@ merge2@^1.3.0, merge2@^1.4.1: resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== +"mi-service-lite@file:../mi-service-lite": + version "2.0.0" + dependencies: + axios "^1.6.5" + pako "^2.1.0" + micromatch@^4.0.4: version "4.0.5" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" @@ -981,6 +987,11 @@ openai@^4.25.0: node-fetch "^2.6.7" web-streams-polyfill "^3.2.1" +pako@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/pako/-/pako-2.1.0.tgz#266cc37f98c7d883545d11335c00fbd4062c9a86" + integrity sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug== + path-key@^3.0.0, path-key@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375"