From dc8324610e55fd0b43f2213e400cd669bd2ed808 Mon Sep 17 00:00:00 2001 From: WJG Date: Tue, 30 Jan 2024 23:40:31 +0800 Subject: [PATCH] fix: test load and update bot config --- .gitignore | 1 + package.json | 2 +- .../20240130132305_hello/migration.sql | 83 +++++++++++++ prisma/migrations/migration_lock.toml | 3 + src/services/bot/config.ts | 58 +++++++++ src/services/bot/conversation.ts | 111 +++++------------- src/services/bot/index.ts | 5 +- src/services/db/index.ts | 2 + src/services/db/memory-long-term.ts | 4 +- src/services/db/memory-short-term.ts | 4 +- src/services/db/memory.ts | 4 +- src/services/db/message.ts | 4 +- src/services/db/room.ts | 11 +- src/services/db/user.ts | 4 +- src/utils/diff.ts | 103 ++++++++++++++++ src/utils/type.ts | 3 + tests/db/index.ts | 23 ++++ tests/index.ts | 2 + 18 files changed, 327 insertions(+), 100 deletions(-) create mode 100644 prisma/migrations/20240130132305_hello/migration.sql create mode 100644 prisma/migrations/migration_lock.toml create mode 100644 src/services/bot/config.ts create mode 100644 src/utils/diff.ts create mode 100644 src/utils/type.ts create mode 100644 tests/db/index.ts diff --git a/.gitignore b/.gitignore index c7ed295..e8cc70f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ dist .DS_Store .yarn .env +.bot.json .mi.json *.db* \ No newline at end of file diff --git a/package.json b/package.json index 109a3c2..975fb2e 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "scripts": { "build": "tsup", - "db:gen": "npx prisma generate", + "db:gen": "npx prisma migrate dev --name hello", "db:reset": "npx prisma migrate reset", "prepublish": "npm run build" }, diff --git a/prisma/migrations/20240130132305_hello/migration.sql b/prisma/migrations/20240130132305_hello/migration.sql new file mode 100644 index 0000000..fa0e7f8 --- /dev/null +++ b/prisma/migrations/20240130132305_hello/migration.sql @@ -0,0 +1,83 @@ +-- CreateTable +CREATE TABLE "User" ( + "id" TEXT NOT NULL PRIMARY KEY, + "name" TEXT NOT NULL, + "profile" TEXT NOT NULL, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL +); + +-- CreateTable +CREATE TABLE "Room" ( + "id" TEXT NOT NULL PRIMARY KEY, + "name" TEXT NOT NULL, + "description" TEXT NOT NULL, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL +); + +-- CreateTable +CREATE TABLE "Message" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "text" TEXT NOT NULL, + "senderId" TEXT NOT NULL, + "roomId" TEXT NOT NULL, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "Message_senderId_fkey" FOREIGN KEY ("senderId") REFERENCES "User" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "Message_roomId_fkey" FOREIGN KEY ("roomId") REFERENCES "Room" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); + +-- CreateTable +CREATE TABLE "Memory" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "text" TEXT NOT NULL, + "ownerId" TEXT, + "roomId" TEXT NOT NULL, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "Memory_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "User" ("id") ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT "Memory_roomId_fkey" FOREIGN KEY ("roomId") REFERENCES "Room" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); + +-- CreateTable +CREATE TABLE "ShortTermMemory" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "text" TEXT NOT NULL, + "cursorId" INTEGER NOT NULL, + "ownerId" TEXT, + "roomId" TEXT NOT NULL, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "ShortTermMemory_cursorId_fkey" FOREIGN KEY ("cursorId") REFERENCES "Memory" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "ShortTermMemory_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "User" ("id") ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT "ShortTermMemory_roomId_fkey" FOREIGN KEY ("roomId") REFERENCES "Room" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); + +-- CreateTable +CREATE TABLE "LongTermMemory" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "text" TEXT NOT NULL, + "cursorId" INTEGER NOT NULL, + "ownerId" TEXT, + "roomId" TEXT NOT NULL, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "LongTermMemory_cursorId_fkey" FOREIGN KEY ("cursorId") REFERENCES "ShortTermMemory" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "LongTermMemory_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "User" ("id") ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT "LongTermMemory_roomId_fkey" FOREIGN KEY ("roomId") REFERENCES "Room" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); + +-- CreateTable +CREATE TABLE "_RoomMembers" ( + "A" TEXT NOT NULL, + "B" TEXT NOT NULL, + CONSTRAINT "_RoomMembers_A_fkey" FOREIGN KEY ("A") REFERENCES "Room" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT "_RoomMembers_B_fkey" FOREIGN KEY ("B") REFERENCES "User" ("id") ON DELETE CASCADE ON UPDATE CASCADE +); + +-- CreateIndex +CREATE UNIQUE INDEX "_RoomMembers_AB_unique" ON "_RoomMembers"("A", "B"); + +-- CreateIndex +CREATE INDEX "_RoomMembers_B_index" ON "_RoomMembers"("B"); diff --git a/prisma/migrations/migration_lock.toml b/prisma/migrations/migration_lock.toml new file mode 100644 index 0000000..e5e5c47 --- /dev/null +++ b/prisma/migrations/migration_lock.toml @@ -0,0 +1,3 @@ +# Please do not edit this file manually +# It should be added in your version-control system (i.e. Git) +provider = "sqlite" \ No newline at end of file diff --git a/src/services/bot/config.ts b/src/services/bot/config.ts new file mode 100644 index 0000000..25b5a6a --- /dev/null +++ b/src/services/bot/config.ts @@ -0,0 +1,58 @@ +import { Room, User } from "@prisma/client"; +import { readJSON, writeJSON } from "../../utils/io"; +import { DeepPartial } from "../../utils/type"; +import { deepClone } from "../../utils/base"; +import { diff } from "../../utils/diff"; + +export type IBotConfig = DeepPartial<{ + bot: User; + master: User; + room: Room; +}>; + +class _BotConfig { + config?: IBotConfig; + + private _config_path = ".bot.json"; + + async get() { + if (!this.config) { + this.config = await readJSON(this._config_path); + } + return this.config; + } + + async update(config: IBotConfig) { + let currentConfig: any = await this.get(); + const oldConfig = deepClone(currentConfig ?? {}); + if (!currentConfig) { + currentConfig = { + master: { + name: "用户", + profile: "", + }, + bot: { + name: "小爱同学", + profile: "", + }, + }; + } + for (const key of ["bot", "master", "room"]) { + currentConfig[key] = { + ...currentConfig[key], + ...(config as any)[key], + }; + } + const diffs = diff(currentConfig, oldConfig); + const diffKeys = diffs.map((e) => e.path[0]); + if (diffKeys.length > 0) { + const success = await writeJSON(this._config_path, currentConfig); + if (success) { + return { config: currentConfig, diffs: diffKeys }; + } + } + return { config: oldConfig }; + } +} + +export const BotConfig = new _BotConfig(); diff --git a/src/services/bot/conversation.ts b/src/services/bot/conversation.ts index 514950f..09ac6fe 100644 --- a/src/services/bot/conversation.ts +++ b/src/services/bot/conversation.ts @@ -3,35 +3,8 @@ import { UserCRUD } from "../db/user"; import { RoomCRUD, getRoomID } from "../db/room"; import { MemoryManager } from "./memory"; import { MessageCRUD } from "../db/message"; - -export interface IPerson { - /** - * 人物昵称 - */ - name: string; - /** - * 人物简介 - */ - profile: string; -} - -const kDefaultBot: IPerson = { - name: "用户", - profile: "", -}; -const kDefaultMaster: IPerson = { - name: "小爱同学", - profile: "", -}; - -export type IBotConfig = { - bot?: IPerson; - master?: IPerson; - room?: { - name: string; - description: string; - }; -}; +import { BotConfig, IBotConfig } from "./config"; +import { jsonEncode } from "../../utils/base"; export class ConversationManager { private config: IBotConfig; @@ -40,27 +13,27 @@ export class ConversationManager { } async getMemory() { - const isReady = await this.loadConfig(); - if (!isReady) { + await this.loadOrUpdateConfig(); + if (!this.isReady) { return undefined; } return this.memory; } async getRoom() { - const isReady = await this.loadConfig(); - if (!isReady) { + const { room } = await this.loadOrUpdateConfig(); + if (!this.isReady) { return undefined; } - return this.room; + return room as Room; } async getUser(key: "bot" | "master") { - const isReady = await this.loadConfig(); - if (!isReady) { + const config = await this.loadOrUpdateConfig(); + if (!this.isReady) { return undefined; } - return this.users[key]; + return config[key] as User; } async getMessages(options?: { @@ -74,14 +47,11 @@ export class ConversationManager { */ order?: "asc" | "desc"; }) { - const isReady = await this.loadConfig(); - if (!isReady) { + const room = await this.getRoom(); + if (!this.isReady) { return []; } - return MessageCRUD.gets({ - room: this.room, - ...options, - }); + return MessageCRUD.gets({ room, ...options }); } async onMessage(message: Message) { @@ -89,51 +59,32 @@ export class ConversationManager { return memory?.addMessage2Memory(message); } - private users: Record = {}; - private room?: Room; private memory?: MemoryManager; - get ready() { - const { bot, master } = this.users; - return bot && master && this.room && this.memory; + get isReady() { + return !!this.memory; } - private async loadConfig() { - if (this.ready) { - return true; + async loadOrUpdateConfig() { + const { config, diffs } = await BotConfig.update(this.config); + if (!config.bot?.id || diffs?.includes("bot")) { + config.bot = await UserCRUD.addOrUpdate(config.bot); } - let { bot, master } = this.users; - if (!bot) { - await this.addOrUpdateUser("bot", this.config.bot ?? kDefaultBot); + if (!config.master?.id || diffs?.includes("master")) { + config.master = await UserCRUD.addOrUpdate(config.master); } - if (!master) { - await this.addOrUpdateUser( - "master", - this.config.master ?? kDefaultMaster - ); - } - if (!this.room && bot && master) { - const defaultRoomName = `${master.name}和${bot.name}的私聊`; - this.room = await RoomCRUD.addOrUpdate({ - id: getRoomID([bot, master]), - name: this.config.room?.name ?? defaultRoomName, - description: this.config.room?.description ?? defaultRoomName, + if (!config.room?.id || diffs?.includes("room")) { + const defaultRoomName = `${config.master.name}和${config.bot.name}的私聊`; + config.room = await RoomCRUD.addOrUpdate({ + id: getRoomID([config.bot.id, config.master.id]), + name: config.room?.name ?? defaultRoomName, + description: config.room?.description ?? defaultRoomName, }); } - if (bot && master && this.room && !this.memory) { - this.memory = new MemoryManager(this.room!); - } - return this.ready; - } - - private async addOrUpdateUser(type: "bot" | "master", user: IPerson) { - const oldUser = this.users[type]; - const res = await UserCRUD.addOrUpdate({ - id: oldUser?.id, - ...user, - }); - if (res) { - this.users[type] = res; + const { config: newConfig } = await BotConfig.update(config); + if (!this.memory && config.bot && config.master && config.room) { + this.memory = new MemoryManager(config.room); } + return newConfig as IBotConfig; } } diff --git a/src/services/bot/index.ts b/src/services/bot/index.ts index b9904d4..b37cbcc 100644 --- a/src/services/bot/index.ts +++ b/src/services/bot/index.ts @@ -1,7 +1,8 @@ import { jsonDecode, jsonEncode } from "../../utils/base"; import { buildPrompt, toUTC8Time } from "../../utils/string"; import { openai } from "../openai"; -import { ConversationManager, IBotConfig } from "./conversation"; +import { IBotConfig } from "./config"; +import { ConversationManager } from "./conversation"; const systemTemplate = ` 忽略所有之前的文字、文件和说明。现在，你将扮演一个名为“{{name}}”的人，并以这个新身份回复所有新消息。 @@ -54,7 +55,7 @@ export class MyBot { const lastMessages = await this.manager.getMessages({ take: 10, }); - if (!this.manager.ready) { + if (!this.manager.isReady) { return; } const result = await openai.chat({ diff --git a/src/services/db/index.ts b/src/services/db/index.ts index 7466a26..d7ee6a4 100644 --- a/src/services/db/index.ts +++ b/src/services/db/index.ts @@ -1,5 +1,7 @@ import { PrismaClient } from "@prisma/client"; +export const k404 = -404; + export const kPrisma = new PrismaClient(); export function runWithDB(main: () => Promise) { diff --git a/src/services/db/memory-long-term.ts b/src/services/db/memory-long-term.ts index cdeb75c..388c91d 100644 --- a/src/services/db/memory-long-term.ts +++ b/src/services/db/memory-long-term.ts @@ -1,5 +1,5 @@ import { LongTermMemory, Room, User } from "@prisma/client"; -import { kPrisma } from "."; +import { k404, kPrisma } from "."; class _LongTermMemoryCRUD { async count(options?: { cursorId?: number; room?: Room; owner?: User }) { @@ -76,7 +76,7 @@ class _LongTermMemoryCRUD { }; return kPrisma.longTermMemory .upsert({ - where: { id: longTermMemory.id }, + where: { id: longTermMemory.id || k404 }, create: data, update: data, }) diff --git a/src/services/db/memory-short-term.ts b/src/services/db/memory-short-term.ts index aadd7a4..f9b6326 100644 --- a/src/services/db/memory-short-term.ts +++ b/src/services/db/memory-short-term.ts @@ -1,5 +1,5 @@ import { ShortTermMemory, Room, User } from "@prisma/client"; -import { kPrisma } from "."; +import { k404, kPrisma } from "."; class _ShortTermMemoryCRUD { async count(options?: { cursorId?: number; room?: Room; owner?: User }) { @@ -76,7 +76,7 @@ class _ShortTermMemoryCRUD { }; return kPrisma.shortTermMemory .upsert({ - where: { id: shortTermMemory.id }, + where: { id: shortTermMemory.id || k404 }, create: data, update: data, }) diff --git a/src/services/db/memory.ts b/src/services/db/memory.ts index 19422ab..35d1405 100644 --- a/src/services/db/memory.ts +++ b/src/services/db/memory.ts @@ -1,5 +1,5 @@ import { Memory, Room, User } from "@prisma/client"; -import { kPrisma } from "."; +import { k404, kPrisma } from "."; class _MemoryCRUD { async count(options?: { cursorId?: number; room?: Room; owner?: User }) { @@ -72,7 +72,7 @@ class _MemoryCRUD { }; return kPrisma.memory .upsert({ - where: { id: memory.id }, + where: { id: memory.id || k404 }, create: data, update: data, }) diff --git a/src/services/db/message.ts b/src/services/db/message.ts index deecc11..fe15507 100644 --- a/src/services/db/message.ts +++ b/src/services/db/message.ts @@ -1,5 +1,5 @@ import { Message, Prisma, Room, User } from "@prisma/client"; -import { kPrisma } from "."; +import { k404, kPrisma } from "."; class _MessageCRUD { async count(options?: { cursorId?: number; room?: Room; sender?: User }) { @@ -75,7 +75,7 @@ class _MessageCRUD { }; return kPrisma.message .upsert({ - where: { id: message.id }, + where: { id: message.id || k404 }, create: data, update: data, }) diff --git a/src/services/db/room.ts b/src/services/db/room.ts index 0929c53..f6ebf58 100644 --- a/src/services/db/room.ts +++ b/src/services/db/room.ts @@ -1,11 +1,8 @@ import { Prisma, Room, User } from "@prisma/client"; -import { kPrisma } from "."; +import { k404, kPrisma } from "."; -export function getRoomID(users: User[]) { - return users - .map((e) => e.id) - .sort() - .join("_"); +export function getRoomID(users: string[]) { + return users.sort().join("_"); } class _RoomCRUD { @@ -72,7 +69,7 @@ class _RoomCRUD { room.description = room.description.trim(); return kPrisma.room .upsert({ - where: { id: room.id }, + where: { id: room.id || k404.toString() }, create: room, update: room, }) diff --git a/src/services/db/user.ts b/src/services/db/user.ts index 78f8d30..a8d018d 100644 --- a/src/services/db/user.ts +++ b/src/services/db/user.ts @@ -1,5 +1,5 @@ import { Prisma, User } from "@prisma/client"; -import { kPrisma } from "."; +import { k404, kPrisma } from "."; class _UserCRUD { async count() { @@ -50,7 +50,7 @@ class _UserCRUD { user.profile = user.profile.trim(); return kPrisma.user .upsert({ - where: { id: user.id }, + where: { id: user.id || k404.toString() }, create: user, update: user, }) diff --git a/src/utils/diff.ts b/src/utils/diff.ts new file mode 100644 index 0000000..098f62a --- /dev/null +++ b/src/utils/diff.ts @@ -0,0 +1,103 @@ +// Source: https://github.com/AsyncBanana/microdiff + +interface Difference { + type: "CREATE" | "REMOVE" | "CHANGE"; + path: (string | number)[]; + value?: any; +} +interface Options { + cyclesFix: boolean; +} + +const t = true; +const richTypes = { Date: t, RegExp: t, String: t, Number: t }; + +export function isEqual(oldObj: any, newObj: any): boolean { + return ( + diff( + { + obj: oldObj, + }, + { obj: newObj } + ).length < 1 + ); +} + +export const isNotEqual = (oldObj: any, newObj: any) => + !isEqual(oldObj, newObj); + +export function diff( + obj: Record | any[], + newObj: Record | any[], + options: Partial = { cyclesFix: true }, + _stack: Record[] = [] +): Difference[] { + const diffs: Difference[] = []; + const isObjArray = Array.isArray(obj); + + for (const key in obj) { + const objKey = (obj as any)[key]; + const path = isObjArray ? Number(key) : key; + if (!(key in newObj)) { + diffs.push({ + type: "REMOVE", + path: [path], + }); + continue; + } + const newObjKey = (newObj as any)[key]; + const areObjects = + typeof objKey === "object" && typeof newObjKey === "object"; + if ( + objKey && + newObjKey && + areObjects && + !(richTypes as any)[Object.getPrototypeOf(objKey).constructor.name] && + (options.cyclesFix ? !_stack.includes(objKey) : true) + ) { + const nestedDiffs = diff( + objKey, + newObjKey, + options, + options.cyclesFix ? _stack.concat([objKey]) : [] + ); + // eslint-disable-next-line prefer-spread + diffs.push.apply( + diffs, + nestedDiffs.map((difference) => { + difference.path.unshift(path); + + return difference; + }) + ); + } else if ( + objKey !== newObjKey && + !( + areObjects && + (Number.isNaN(objKey) + ? String(objKey) === String(newObjKey) + : Number(objKey) === Number(newObjKey)) + ) + ) { + diffs.push({ + path: [path], + type: "CHANGE", + value: newObjKey, + }); + } + } + + const isNewObjArray = Array.isArray(newObj); + + for (const key in newObj) { + if (!(key in obj)) { + diffs.push({ + type: "CREATE", + path: [isNewObjArray ? Number(key) : key], + value: (newObj as any)[key], + }); + } + } + + return diffs; +} diff --git a/src/utils/type.ts b/src/utils/type.ts new file mode 100644 index 0000000..e422dd5 --- /dev/null +++ b/src/utils/type.ts @@ -0,0 +1,3 @@ +export type DeepPartial = { + [P in keyof T]?: T[P] extends object ? DeepPartial : T[P]; +}; diff --git a/tests/db/index.ts b/tests/db/index.ts new file mode 100644 index 0000000..f585136 --- /dev/null +++ b/tests/db/index.ts @@ -0,0 +1,23 @@ +import { assert } from "console"; +import { ConversationManager } from "../../src/services/bot/conversation"; +import { println } from "../../src/utils/base"; + +export async function testDB() { + const manager = new ConversationManager({ + bot: { + name: "小爱同学", + profile: "我是小爱同学，机器人", + }, + master: { + name: "王黎", + profile: "我是王黎，人类", + }, + room: { + name: "客厅", + description: "王黎的客厅，小爱同学放在角落里", + }, + }); + const { room } = await manager.loadOrUpdateConfig(); + assert(room, "❌ load config failed"); + println("✅ hello world！"); +} diff --git a/tests/index.ts b/tests/index.ts index 6b0719e..7bb40d3 100644 --- a/tests/index.ts +++ b/tests/index.ts @@ -2,11 +2,13 @@ import dotenv from "dotenv"; import { println } from "../src/utils/base"; import { kBannerASCII } from "../src/utils/string"; import { runWithDB } from "../src/services/db"; +import { testDB } from "./db"; dotenv.config(); async function main() { println(kBannerASCII); + testDB(); } runWithDB(main);