feat: add xiaoai speaker module

This commit is contained in:
WJG 2024-02-23 18:10:53 +08:00
parent 18c246ac6c
commit e948261e4e
No known key found for this signature in database
GPG Key ID: 258474EF8590014A
10 changed files with 730 additions and 6 deletions

3
TODO.md Normal file
View File

@ -0,0 +1,3 @@
- ❌ Auto mute XiaoAi reply
- Stream response
- Update long/short memories

View File

@ -1,6 +1,6 @@
{
"name": "mi-gpt",
"version": "0.1.0",
"version": "0.2.0",
"type": "module",
"description": "Seamlessly integrate your XiaoAI speaker and Mi Home devices with ChatGPT for an enhanced smart home experience.",
"license": "MIT",
@ -27,6 +27,7 @@
"dependencies": {
"@prisma/client": "^5.8.1",
"axios": "^1.6.5",
"mi-service-lite": "^2.0.0",
"openai": "^4.25.0",
"prisma": "^5.8.1"
},

220
src/services/speaker/ai.ts Normal file
View File

@ -0,0 +1,220 @@
import { pickOne } from "../../utils/base";
import {
Speaker,
SpeakerCommand,
SpeakerConfig,
QueryMessage,
} from "./speaker";
export type AISpeakerConfig = SpeakerConfig & {
askAI?: (msg: QueryMessage) => Promise<string>;
/**
*
*
*
*/
switchSpeakerPrefix?: string;
/**
* AI
*
*
*/
onAIAsking?: string[];
/**
* AI
*
*
*/
onAIError?: string[];
/**
* /退
*
* 使使
*/
name?: string;
/**
*
*
* AI
*
* //
*/
callAIPrefix?: string[];
/**
*
*
* AI
*
* /退/
*/
wakeUpKeyWords?: string[];
/**
* 退
*
* 退退 AI
*/
exitKeywords?: string[];
/**
* AI
*
*
*/
onEnterAI?: string[];
/**
* 退 AI
*
* 退
*/
onExitAI?: string[];
};
type AnswerStep = (
msg: any,
data: any
) => Promise<{ stop?: boolean; data?: any } | void>;
export class AISpeaker extends Speaker {
askAI: AISpeakerConfig["askAI"];
name: string;
switchSpeakerPrefix: string;
onEnterAI: string[];
onExitAI: string[];
callAIPrefix: string[];
wakeUpKeyWords: string[];
exitKeywords: string[];
onAIAsking: string[];
onAIError: string[];
constructor(config: AISpeakerConfig) {
super(config);
const {
askAI,
name = "豆包",
switchSpeakerPrefix = "音色切换到",
wakeUpKeyWords = ["打开", "进入", "召唤"],
exitKeywords = ["关闭", "退出", "再见"],
onAIAsking = ["让我先想想", "请稍等"],
onAIError = ["啊哦,出错了,请稍后再试吧!"],
} = config;
this.askAI = askAI;
this.switchSpeakerPrefix = switchSpeakerPrefix;
this.name = name;
this.onAIError = onAIError;
this.onAIAsking = onAIAsking;
this.wakeUpKeyWords = wakeUpKeyWords.map((e) => e + this.name);
this.exitKeywords = exitKeywords.map((e) => e + this.name);
this.onEnterAI = config.onEnterAI ?? [
`你好,我是${this.name},很高兴为你服务!`,
];
this.onExitAI = config.onExitAI ?? [`${this.name}已关闭!`];
this.callAIPrefix = config.callAIPrefix ?? [
"请",
"你",
this.name,
"问问" + this.name,
];
}
async enterKeepAlive() {
// 回应
await this.response({ text: pickOne(this.onEnterAI)!, keepAlive: true });
// 唤醒
await super.enterKeepAlive();
}
async exitKeepAlive() {
// 退出唤醒状态
await super.exitKeepAlive();
// 回应
await this.response({
text: pickOne(this.onExitAI)!,
keepAlive: false,
playSFX: false,
});
await this.unWakeUp();
}
get commands() {
return [
{
match: (msg) => this.wakeUpKeyWords.some((e) => msg.text.includes(e)),
run: async (msg) => {
await this.enterKeepAlive();
},
},
{
match: (msg) => this.exitKeywords.some((e) => msg.text.includes(e)),
run: async (msg) => {
await this.exitKeepAlive();
},
},
{
match: (msg) => msg.text.startsWith(this.switchSpeakerPrefix),
run: async (msg) => {
await this.response({
text: "正在切换音色,请稍等...",
keepAlive: this.keepAlive,
});
const speaker = msg.text.replace(this.switchSpeakerPrefix, "");
const success = await this.switchDefaultSpeaker(speaker);
await this.response({
text: success ? "音色已切换!" : "音色切换失败!",
keepAlive: this.keepAlive,
});
},
},
...this._commands,
{
match: (msg) =>
this.keepAlive ||
this.callAIPrefix.some((e) => msg.text.startsWith(e)),
run: (msg) => this.askAIForAnswer(msg),
},
] as SpeakerCommand[];
}
private _askAIForAnswerSteps: AnswerStep[] = [
async (msg, data) => {
// 思考中
await this.response({
audio: process.env.AUDIO_ACTIVE,
text: pickOne(this.onAIAsking)!,
keepAlive: this.keepAlive,
});
},
async (msg, data) => {
// 调用 AI 获取回复
let answer = await this.askAI?.(msg);
return { data: { answer } };
},
async (msg, data) => {
if (!data.answer) {
// 回答异常
await this.response({
audio: process.env.AUDIO_ERROR,
text: pickOne(this.onAIError)!,
keepAlive: this.keepAlive,
});
}
},
];
async askAIForAnswer(msg: QueryMessage) {
let data: any = {};
const { hasNewMsg } = this.checkIfHasNewMsg(msg);
for (const action of this._askAIForAnswerSteps) {
const res = await action(msg, data);
if (hasNewMsg()) {
// 收到新的用户请求消息,终止后续操作和响应
return;
}
if (res?.data) {
data = { ...data, ...res.data };
}
if (res?.stop) {
break;
}
}
return data.answer;
}
}

View File

@ -0,0 +1,158 @@
import { assert } from "console";
import {
MiServiceConfig,
getMiIOT,
getMiNA,
MiNA,
MiIOT,
} from "mi-service-lite";
import { sleep } from "../../utils/base";
import { Http } from "../http";
export type TTSProvider = "xiaoai" | "doubao";
type Speaker = {
name: string;
gender: "男" | "女";
speaker: string;
};
export type BaseSpeakerConfig = MiServiceConfig & {
// 语音合成服务商
tts?: TTSProvider;
// 检测间隔(单位毫秒,默认 100 毫秒)
interval?: number;
};
export class BaseSpeaker {
MiNA?: MiNA;
MiIOT?: MiIOT;
interval: number;
tts: TTSProvider;
config: MiServiceConfig;
constructor(config: BaseSpeakerConfig) {
this.config = config;
const { interval = 100, tts = "doubao" } = config;
this.interval = interval;
this.tts = tts;
}
async initMiServices() {
this.MiNA = await getMiNA(this.config);
this.MiIOT = await getMiIOT(this.config);
assert(!!this.MiNA && !!this.MiIOT, "❌ init Mi Services failed");
}
wakeUp() {
return this.MiIOT!.doAction(5, 3);
}
async unWakeUp() {
await this.MiIOT!.setProperty(4, 1, true); // 关闭麦克风
await this.MiIOT!.setProperty(4, 1, false); // 打开麦克风
}
responding = false;
async response(options: {
tts?: TTSProvider;
text?: string;
audio?: string;
speaker?: string;
keepAlive?: boolean;
playSFX?: boolean;
}) {
let {
text,
audio,
playSFX = true,
keepAlive = false,
tts = this.tts,
speaker = this._defaultSpeaker,
} = options ?? {};
// 播放回复
const play = async (args?: { tts?: string; url?: string }) => {
const ttsNotXiaoai = !audio && tts !== "xiaoai";
playSFX = ttsNotXiaoai && playSFX;
// 播放开始提示音
if (playSFX) {
await this.MiNA!.play({ url: process.env.AUDIO_BEEP });
}
// 在播放 TTS 语音之前,先取消小爱音箱的唤醒状态,防止将 TTS 语音识别成用户指令
if (ttsNotXiaoai) {
await this.unWakeUp();
}
await this.MiNA!.play(args);
console.log("✅ " + text ?? audio);
// 等待回答播放完毕
while (true) {
const res = await this.MiNA!.getStatus();
if (
!this.responding || // 有新消息
(res?.status === "playing" && res?.media_type) // 小爱自己开始播放音乐
) {
// 响应被中断
return "break";
}
if (res?.status && res.status !== "playing") {
break;
}
await sleep(this.interval);
}
// 播放结束提示音
if (playSFX) {
await this.MiNA!.play({ url: process.env.AUDIO_BEEP });
}
// 保持唤醒状态
if (keepAlive) {
await this.wakeUp();
}
};
// 开始响应
let res;
this.responding = true;
if (audio) {
// 音频回复
res = await play({ url: audio });
} else if (text) {
// 文字回复
switch (tts) {
case "doubao":
text = encodeURIComponent(text);
const doubaoTTS = process.env.TTS_DOUBAO;
const url = `${doubaoTTS}?speaker=${speaker}&text=${text}`;
res = await play({ url });
break;
case "xiaoai":
default:
res = await play({ tts: text });
}
this.responding = false;
return res;
}
}
private _doubaoSpeakers?: Speaker[];
private _defaultSpeaker = "zh_female_maomao_conversation_wvae_bigtts";
async switchDefaultSpeaker(speaker: string) {
if (!this._doubaoSpeakers) {
const doubaoSpeakers = process.env.SPEAKERS_DOUBAO;
const res = await Http.get(doubaoSpeakers ?? "/");
if (Array.isArray(res)) {
this._doubaoSpeakers = res;
}
}
if (!this._doubaoSpeakers) {
return false;
}
const target = this._doubaoSpeakers.find(
(e) => e.name === speaker || e.speaker === speaker
);
if (target) {
this._defaultSpeaker = target.speaker;
}
return this._defaultSpeaker === target?.speaker;
}
}

View File

@ -0,0 +1,274 @@
import { firstOf, lastOf, sleep } from "../../utils/base";
import { BaseSpeaker, BaseSpeakerConfig } from "./base";
export interface QueryMessage {
text: string;
answer: string;
/**
*
*/
timestamp: number;
}
export interface SpeakerCommand {
match: (msg: QueryMessage) => boolean;
/**
*
*/
run: (msg: QueryMessage) => Promise<string | undefined | void>;
}
export type SpeakerConfig = BaseSpeakerConfig & {
/**
* 1
*/
heartbeat?: number;
/**
*
*/
commands?: SpeakerCommand[];
/**
* 退30
*/
exitKeepAliveAfter?: number;
};
export class Speaker extends BaseSpeaker {
heartbeat: number;
exitKeepAliveAfter: number;
currentQueryMsg?: QueryMessage;
constructor(config: SpeakerConfig) {
super(config);
const { heartbeat = 1000, exitKeepAliveAfter = 30 } = config;
this._commands = config.commands ?? [];
this.heartbeat = heartbeat;
this.exitKeepAliveAfter = exitKeepAliveAfter;
}
private _status: "running" | "stopped" = "running";
stop() {
this._status = "stopped";
}
async run() {
await this.initMiServices();
if (!this.MiNA) {
this.stop();
}
console.log("✅ 服务已启动...");
this.activeKeepAliveMode();
while (this._status === "running") {
const nextMsg = await this.fetchNextMessage();
if (nextMsg) {
this.responding = false;
console.log("🔥 " + nextMsg.text);
// 异步处理消息,不阻塞正常消息拉取
this.onMessage(nextMsg);
}
await sleep(this.heartbeat);
}
}
async activeKeepAliveMode() {
while (this._status === "running") {
if (this.keepAlive) {
// 唤醒中
if (!this.responding) {
// 没有回复时,一直播放静音音频使小爱闭嘴
await this.MiNA?.play({ url: process.env.AUDIO_SILENT });
}
}
await sleep(this.interval);
}
}
_commands: SpeakerCommand[] = [];
get commands() {
return this._commands;
}
addCommand(command: SpeakerCommand) {
this.commands.push(command);
}
async onMessage(msg: QueryMessage) {
const { noNewMsg } = this.checkIfHasNewMsg(msg);
for (const command of this.commands) {
if (command.match(msg)) {
// 关闭小爱的回复
await this.MiNA!.pause();
// 执行命令
const answer = await command.run(msg);
// 回复用户
if (answer) {
if (noNewMsg()) {
await this.response({
text: answer,
keepAlive: this.keepAlive,
});
}
}
await this.exitKeepAliveIfNeeded();
return;
}
}
}
/**
*
*/
keepAlive = false;
async enterKeepAlive() {
// 唤醒
this.keepAlive = true;
}
async exitKeepAlive() {
// 退出唤醒状态
this.keepAlive = false;
}
private _preTimer: any;
async exitKeepAliveIfNeeded() {
// 无响应一段时间后自动退出唤醒状态
if (this._preTimer) {
clearTimeout(this._preTimer);
}
const { noNewMsg } = this.checkIfHasNewMsg();
this._preTimer = setTimeout(async () => {
if (this.keepAlive && !this.responding && noNewMsg()) {
await this.exitKeepAlive();
}
}, this.exitKeepAliveAfter * 1000);
}
checkIfHasNewMsg(currentMsg?: QueryMessage) {
const currentTimestamp = (currentMsg ?? this.currentQueryMsg)?.timestamp;
return {
hasNewMsg: () => currentTimestamp !== this.currentQueryMsg?.timestamp,
noNewMsg: () => currentTimestamp === this.currentQueryMsg?.timestamp,
};
}
private _tempMsgs: QueryMessage[] = [];
async fetchNextMessage(): Promise<QueryMessage | undefined> {
if (!this.currentQueryMsg) {
await this._fetchFirstMessage();
// 第一条消息仅用作初始化消息游标,不响应
return;
}
return this._fetchNextMessage();
}
private async _fetchFirstMessage() {
const msgs = await this.getMessages({
limit: 1,
filterTTS: false,
});
this.currentQueryMsg = msgs[0];
}
private async _fetchNextMessage(): Promise<QueryMessage | undefined> {
if (this._tempMsgs.length > 0) {
// 当前有暂存的新消息(从新到旧),依次处理之
return this._fetchNextTempMessage();
}
// 拉取最新的 2 条 msg用于和上一条消息比对是否连续
const nextMsg = await this._fetchNext2Messages();
if (nextMsg !== "continue") {
return nextMsg;
}
// 继续向上拉取其他新消息
return this._fetchNextRemainingMessages();
}
private async _fetchNext2Messages() {
// 拉取最新的 2 条 msg用于和上一条消息比对是否连续
let msgs = await this.getMessages({ limit: 2 });
if (
msgs.length < 1 ||
firstOf(msgs)!.timestamp <= this.currentQueryMsg!.timestamp
) {
// 没有拉到新消息
return;
}
if (
firstOf(msgs)!.timestamp > this.currentQueryMsg!.timestamp &&
(msgs.length === 1 ||
lastOf(msgs)!.timestamp <= this.currentQueryMsg!.timestamp)
) {
// 刚好收到一条新消息
this.currentQueryMsg = firstOf(msgs);
return this.currentQueryMsg;
}
// 还有其他新消息,暂存当前的新消息
for (const msg of msgs) {
if (msg.timestamp > this.currentQueryMsg!.timestamp) {
this._tempMsgs.push(msg);
}
}
return "continue";
}
private _fetchNextTempMessage() {
const nextMsg = this._tempMsgs.pop();
this.currentQueryMsg = nextMsg;
return nextMsg;
}
private async _fetchNextRemainingMessages(maxPage = 3) {
// 继续向上拉取其他新消息
let currentPage = 0;
while (true) {
currentPage++;
if (currentPage > maxPage) {
// 拉取新消息超长,取消拉取
return this._fetchNextTempMessage();
}
const nextTimestamp = lastOf(this._tempMsgs)!.timestamp;
const msgs = await this.getMessages({
limit: 10,
timestamp: nextTimestamp,
});
for (const msg of msgs) {
if (msg.timestamp >= nextTimestamp) {
// 忽略上一页的消息
continue;
} else if (msg.timestamp > this.currentQueryMsg!.timestamp) {
// 继续添加新消息
this._tempMsgs.push(msg);
} else {
// 拉取到历史消息处
return this._fetchNextTempMessage();
}
}
}
}
async getMessages(options?: {
limit?: number;
timestamp?: number;
filterTTS?: boolean;
}): Promise<QueryMessage[]> {
const filterTTS = options?.filterTTS ?? true;
const conversation = await this.MiNA!.getConversations(options);
let records = conversation?.records ?? [];
if (filterTTS) {
// 过滤有小爱回答的消息
records = records.filter(
(e) => e.answers.length > 0 && e.answers.some((e) => e.type === "TTS")
);
}
return records.map((e) => {
const ttsAnswer = e.answers.find((e) => e.type === "TTS") as any;
return {
text: e.query,
answer: ttsAnswer?.tts?.text,
timestamp: e.time,
};
});
}
}

View File

@ -4,7 +4,7 @@ export function timestamp() {
return new Date().getTime();
}
export async function delay(time: number) {
export async function sleep(time: number) {
return new Promise<void>((resolve) => setTimeout(resolve, time));
}

View File

@ -1,7 +1,7 @@
import { assert } from "console";
import { ConversationManager } from "../../src/services/bot/conversation";
import { println } from "../../src/utils/base";
import { MessageCRUD } from "../../src/services/db/message";
import { ConversationManager } from "../src/services/bot/conversation";
import { println } from "../src/utils/base";
import { MessageCRUD } from "../src/services/db/message";
export async function testDB() {
const manager = new ConversationManager({

58
tests/speaker.ts Normal file
View File

@ -0,0 +1,58 @@
import { AISpeaker } from "../src/services/speaker/ai";
import { sleep } from "../src/utils/base";
export async function main() {
const config: any = {
userId: process.env.MI_USER!,
password: process.env.MI_PASS!,
did: process.env.MI_DID,
tts: "doubao",
};
const speaker = new AISpeaker(config);
await speaker.initMiServices();
// await testSpeakerResponse(speaker);
// await testSpeakerGetMessages(speaker);
// await testSwitchSpeaker(speaker);
// await testSpeakerUnWakeUp(speaker);
await testAISpeaker(speaker);
}
async function testAISpeaker(speaker: AISpeaker) {
speaker.askAI = async (msg) => {
return "你说:" + msg.text;
};
await speaker.run();
console.log("finished");
}
async function testSpeakerUnWakeUp(speaker: AISpeaker) {
await speaker.wakeUp();
await sleep(1000);
await speaker.unWakeUp();
console.log("hello");
}
async function testSwitchSpeaker(speaker: AISpeaker) {
await speaker.response({ text: "你好,我是豆包,很高兴认识你!" });
const success = await speaker.switchDefaultSpeaker("魅力苏菲");
console.log("switchDefaultSpeaker 魅力苏菲", success);
await speaker.response({ text: "你好,我是豆包,很高兴认识你!" });
console.log("hello");
}
async function testSpeakerGetMessages(speaker: AISpeaker) {
let msgs = await speaker.getMessages({ filterTTS: true });
console.log("filterTTS msgs", msgs);
msgs = await speaker.getMessages({ filterTTS: false });
console.log("no filterTTS msgs", msgs);
}
async function testSpeakerResponse(speaker: AISpeaker) {
let status = await speaker.MiNA!.getStatus();
console.log("curent status", status);
speaker.response({ text: "你好,我是豆包,很高兴认识你!" });
sleep(1000);
status = await speaker.MiNA!.getStatus();
console.log("tts status", status);
}

View File

@ -6,7 +6,6 @@ export default defineConfig(() => ({
target: "node16",
platform: "node",
format: ["esm", "cjs"],
splitting: false,
sourcemap: false,
treeshake: true,
minify: true,

View File

@ -874,6 +874,12 @@ merge2@^1.3.0, merge2@^1.4.1:
resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae"
integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==
"mi-service-lite@file:../mi-service-lite":
version "2.0.0"
dependencies:
axios "^1.6.5"
pako "^2.1.0"
micromatch@^4.0.4:
version "4.0.5"
resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6"
@ -981,6 +987,11 @@ openai@^4.25.0:
node-fetch "^2.6.7"
web-streams-polyfill "^3.2.1"
pako@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/pako/-/pako-2.1.0.tgz#266cc37f98c7d883545d11335c00fbd4062c9a86"
integrity sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==
path-key@^3.0.0, path-key@^3.1.0:
version "3.1.1"
resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375"