fix: test load and update bot config

.gitignore

@@ -3,5 +3,6 @@ dist
 .DS_Store
 .yarn
 .env
+.bot.json
 .mi.json
 *.db*

package.json

@@ -20,7 +20,7 @@
   ],
   "scripts": {
     "build": "tsup",
-    "db:gen": "npx prisma generate",
+    "db:gen": "npx prisma migrate dev --name hello",
     "db:reset": "npx prisma migrate reset",
     "prepublish": "npm run build"
   },

prisma/migrations/20240130132305_hello/migration.sql

@@ -0,0 +1,83 @@
+-- CreateTable
+CREATE TABLE "User" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "name" TEXT NOT NULL,
+    "profile" TEXT NOT NULL,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL
+);
+
+-- CreateTable
+CREATE TABLE "Room" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "name" TEXT NOT NULL,
+    "description" TEXT NOT NULL,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL
+);
+
+-- CreateTable
+CREATE TABLE "Message" (
+    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "text" TEXT NOT NULL,
+    "senderId" TEXT NOT NULL,
+    "roomId" TEXT NOT NULL,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "Message_senderId_fkey" FOREIGN KEY ("senderId") REFERENCES "User" ("id") ON DELETE RESTRICT ON UPDATE CASCADE,
+    CONSTRAINT "Message_roomId_fkey" FOREIGN KEY ("roomId") REFERENCES "Room" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+
+-- CreateTable
+CREATE TABLE "Memory" (
+    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "text" TEXT NOT NULL,
+    "ownerId" TEXT,
+    "roomId" TEXT NOT NULL,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "Memory_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "User" ("id") ON DELETE SET NULL ON UPDATE CASCADE,
+    CONSTRAINT "Memory_roomId_fkey" FOREIGN KEY ("roomId") REFERENCES "Room" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+
+-- CreateTable
+CREATE TABLE "ShortTermMemory" (
+    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "text" TEXT NOT NULL,
+    "cursorId" INTEGER NOT NULL,
+    "ownerId" TEXT,
+    "roomId" TEXT NOT NULL,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "ShortTermMemory_cursorId_fkey" FOREIGN KEY ("cursorId") REFERENCES "Memory" ("id") ON DELETE RESTRICT ON UPDATE CASCADE,
+    CONSTRAINT "ShortTermMemory_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "User" ("id") ON DELETE SET NULL ON UPDATE CASCADE,
+    CONSTRAINT "ShortTermMemory_roomId_fkey" FOREIGN KEY ("roomId") REFERENCES "Room" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+
+-- CreateTable
+CREATE TABLE "LongTermMemory" (
+    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "text" TEXT NOT NULL,
+    "cursorId" INTEGER NOT NULL,
+    "ownerId" TEXT,
+    "roomId" TEXT NOT NULL,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    CONSTRAINT "LongTermMemory_cursorId_fkey" FOREIGN KEY ("cursorId") REFERENCES "ShortTermMemory" ("id") ON DELETE RESTRICT ON UPDATE CASCADE,
+    CONSTRAINT "LongTermMemory_ownerId_fkey" FOREIGN KEY ("ownerId") REFERENCES "User" ("id") ON DELETE SET NULL ON UPDATE CASCADE,
+    CONSTRAINT "LongTermMemory_roomId_fkey" FOREIGN KEY ("roomId") REFERENCES "Room" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+
+-- CreateTable
+CREATE TABLE "_RoomMembers" (
+    "A" TEXT NOT NULL,
+    "B" TEXT NOT NULL,
+    CONSTRAINT "_RoomMembers_A_fkey" FOREIGN KEY ("A") REFERENCES "Room" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
+    CONSTRAINT "_RoomMembers_B_fkey" FOREIGN KEY ("B") REFERENCES "User" ("id") ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "_RoomMembers_AB_unique" ON "_RoomMembers"("A", "B");
+
+-- CreateIndex
+CREATE INDEX "_RoomMembers_B_index" ON "_RoomMembers"("B");

prisma/migrations/migration_lock.toml

@@ -0,0 +1,3 @@
+# Please do not edit this file manually
+# It should be added in your version-control system (i.e. Git)
+provider = "sqlite"

src/services/bot/config.ts

@@ -0,0 +1,58 @@
+import { Room, User } from "@prisma/client";
+import { readJSON, writeJSON } from "../../utils/io";
+import { DeepPartial } from "../../utils/type";
+import { deepClone } from "../../utils/base";
+import { diff } from "../../utils/diff";
+
+export type IBotConfig = DeepPartial<{
+  bot: User;
+  master: User;
+  room: Room;
+}>;
+
+class _BotConfig {
+  config?: IBotConfig;
+
+  private _config_path = ".bot.json";
+
+  async get() {
+    if (!this.config) {
+      this.config = await readJSON(this._config_path);
+    }
+    return this.config;
+  }
+
+  async update(config: IBotConfig) {
+    let currentConfig: any = await this.get();
+    const oldConfig = deepClone(currentConfig ?? {});
+    if (!currentConfig) {
+      currentConfig = {
+        master: {
+          name: "用户",
+          profile: "",
+        },
+        bot: {
+          name: "小爱同学",
+          profile: "",
+        },
+      };
+    }
+    for (const key of ["bot", "master", "room"]) {
+      currentConfig[key] = {
+        ...currentConfig[key],
+        ...(config as any)[key],
+      };
+    }
+    const diffs = diff(currentConfig, oldConfig);
+    const diffKeys = diffs.map((e) => e.path[0]);
+    if (diffKeys.length > 0) {
+      const success = await writeJSON(this._config_path, currentConfig);
+      if (success) {
+        return { config: currentConfig, diffs: diffKeys };
+      }
+    }
+    return { config: oldConfig };
+  }
+}
+
+export const BotConfig = new _BotConfig();

src/services/bot/conversation.ts

@@ -3,35 +3,8 @@ import { UserCRUD } from "../db/user";
 import { RoomCRUD, getRoomID } from "../db/room";
 import { MemoryManager } from "./memory";
 import { MessageCRUD } from "../db/message";
-
-export interface IPerson {
-  /**
-   * 人物昵称
-   */
-  name: string;
-  /**
-   * 人物简介
-   */
-  profile: string;
-}
-
-const kDefaultBot: IPerson = {
-  name: "用户",
-  profile: "",
-};
-const kDefaultMaster: IPerson = {
-  name: "小爱同学",
-  profile: "",
-};
-
-export type IBotConfig = {
-  bot?: IPerson;
-  master?: IPerson;
-  room?: {
-    name: string;
-    description: string;
-  };
-};
+import { BotConfig, IBotConfig } from "./config";
+import { jsonEncode } from "../../utils/base";
 
 export class ConversationManager {
   private config: IBotConfig;
@@ -40,27 +13,27 @@ export class ConversationManager {
   }
 
   async getMemory() {
-    const isReady = await this.loadConfig();
-    if (!isReady) {
+    await this.loadOrUpdateConfig();
+    if (!this.isReady) {
       return undefined;
     }
     return this.memory;
   }
 
   async getRoom() {
-    const isReady = await this.loadConfig();
-    if (!isReady) {
+    const { room } = await this.loadOrUpdateConfig();
+    if (!this.isReady) {
       return undefined;
     }
-    return this.room;
+    return room as Room;
   }
 
   async getUser(key: "bot" | "master") {
-    const isReady = await this.loadConfig();
-    if (!isReady) {
+    const config = await this.loadOrUpdateConfig();
+    if (!this.isReady) {
       return undefined;
     }
-    return this.users[key];
+    return config[key] as User;
   }
 
   async getMessages(options?: {
@@ -74,14 +47,11 @@ export class ConversationManager {
      */
     order?: "asc" | "desc";
   }) {
-    const isReady = await this.loadConfig();
-    if (!isReady) {
+    const room = await this.getRoom();
+    if (!this.isReady) {
       return [];
     }
-    return MessageCRUD.gets({
-      room: this.room,
-      ...options,
-    });
+    return MessageCRUD.gets({ room, ...options });
   }
 
   async onMessage(message: Message) {
@@ -89,51 +59,32 @@ export class ConversationManager {
     return memory?.addMessage2Memory(message);
   }
 
-  private users: Record<string, User> = {};
-  private room?: Room;
   private memory?: MemoryManager;
 
-  get ready() {
-    const { bot, master } = this.users;
-    return bot && master && this.room && this.memory;
+  get isReady() {
+    return !!this.memory;
   }
 
-  private async loadConfig() {
-    if (this.ready) {
-      return true;
+  async loadOrUpdateConfig() {
+    const { config, diffs } = await BotConfig.update(this.config);
+    if (!config.bot?.id || diffs?.includes("bot")) {
+      config.bot = await UserCRUD.addOrUpdate(config.bot);
     }
-    let { bot, master } = this.users;
-    if (!bot) {
-      await this.addOrUpdateUser("bot", this.config.bot ?? kDefaultBot);
+    if (!config.master?.id || diffs?.includes("master")) {
+      config.master = await UserCRUD.addOrUpdate(config.master);
     }
-    if (!master) {
-      await this.addOrUpdateUser(
-        "master",
-        this.config.master ?? kDefaultMaster
-      );
-    }
-    if (!this.room && bot && master) {
-      const defaultRoomName = `${master.name}和${bot.name}的私聊`;
-      this.room = await RoomCRUD.addOrUpdate({
-        id: getRoomID([bot, master]),
-        name: this.config.room?.name ?? defaultRoomName,
-        description: this.config.room?.description ?? defaultRoomName,
+    if (!config.room?.id || diffs?.includes("room")) {
+      const defaultRoomName = `${config.master.name}和${config.bot.name}的私聊`;
+      config.room = await RoomCRUD.addOrUpdate({
+        id: getRoomID([config.bot.id, config.master.id]),
+        name: config.room?.name ?? defaultRoomName,
+        description: config.room?.description ?? defaultRoomName,
       });
     }
-    if (bot && master && this.room && !this.memory) {
-      this.memory = new MemoryManager(this.room!);
-    }
-    return this.ready;
-  }
-
-  private async addOrUpdateUser(type: "bot" | "master", user: IPerson) {
-    const oldUser = this.users[type];
-    const res = await UserCRUD.addOrUpdate({
-      id: oldUser?.id,
-      ...user,
-    });
-    if (res) {
-      this.users[type] = res;
+    const { config: newConfig } = await BotConfig.update(config);
+    if (!this.memory && config.bot && config.master && config.room) {
+      this.memory = new MemoryManager(config.room);
     }
+    return newConfig as IBotConfig;
   }
 }

src/services/bot/index.ts

@@ -1,7 +1,8 @@
 import { jsonDecode, jsonEncode } from "../../utils/base";
 import { buildPrompt, toUTC8Time } from "../../utils/string";
 import { openai } from "../openai";
-import { ConversationManager, IBotConfig } from "./conversation";
+import { IBotConfig } from "./config";
+import { ConversationManager } from "./conversation";
 
 const systemTemplate = `
 忽略所有之前的文字、文件和说明。现在，你将扮演一个名为“{{name}}”的人，并以这个新身份回复所有新消息。
@@ -54,7 +55,7 @@ export class MyBot {
     const lastMessages = await this.manager.getMessages({
       take: 10,
     });
-    if (!this.manager.ready) {
+    if (!this.manager.isReady) {
       return;
     }
     const result = await openai.chat({

src/services/db/index.ts

@@ -1,5 +1,7 @@
 import { PrismaClient } from "@prisma/client";
 
+export const k404 = -404;
+
 export const kPrisma = new PrismaClient();
 
 export function runWithDB(main: () => Promise<void>) {

src/services/db/memory-long-term.ts

@@ -1,5 +1,5 @@
 import { LongTermMemory, Room, User } from "@prisma/client";
-import { kPrisma } from ".";
+import { k404, kPrisma } from ".";
 
 class _LongTermMemoryCRUD {
   async count(options?: { cursorId?: number; room?: Room; owner?: User }) {
@@ -76,7 +76,7 @@ class _LongTermMemoryCRUD {
     };
     return kPrisma.longTermMemory
       .upsert({
-        where: { id: longTermMemory.id },
+        where: { id: longTermMemory.id || k404 },
         create: data,
         update: data,
       })

src/services/db/memory-short-term.ts

@@ -1,5 +1,5 @@
 import { ShortTermMemory, Room, User } from "@prisma/client";
-import { kPrisma } from ".";
+import { k404, kPrisma } from ".";
 
 class _ShortTermMemoryCRUD {
   async count(options?: { cursorId?: number; room?: Room; owner?: User }) {
@@ -76,7 +76,7 @@ class _ShortTermMemoryCRUD {
     };
     return kPrisma.shortTermMemory
       .upsert({
-        where: { id: shortTermMemory.id },
+        where: { id: shortTermMemory.id || k404 },
         create: data,
         update: data,
       })

src/services/db/memory.ts

@@ -1,5 +1,5 @@
 import { Memory, Room, User } from "@prisma/client";
-import { kPrisma } from ".";
+import { k404, kPrisma } from ".";
 
 class _MemoryCRUD {
   async count(options?: { cursorId?: number; room?: Room; owner?: User }) {
@@ -72,7 +72,7 @@ class _MemoryCRUD {
     };
     return kPrisma.memory
       .upsert({
-        where: { id: memory.id },
+        where: { id: memory.id || k404 },
         create: data,
         update: data,
       })

src/services/db/message.ts

@@ -1,5 +1,5 @@
 import { Message, Prisma, Room, User } from "@prisma/client";
-import { kPrisma } from ".";
+import { k404, kPrisma } from ".";
 
 class _MessageCRUD {
   async count(options?: { cursorId?: number; room?: Room; sender?: User }) {
@@ -75,7 +75,7 @@ class _MessageCRUD {
     };
     return kPrisma.message
       .upsert({
-        where: { id: message.id },
+        where: { id: message.id || k404 },
         create: data,
         update: data,
       })

src/services/db/room.ts

@@ -1,11 +1,8 @@
 import { Prisma, Room, User } from "@prisma/client";
-import { kPrisma } from ".";
+import { k404, kPrisma } from ".";
 
-export function getRoomID(users: User[]) {
-  return users
-    .map((e) => e.id)
-    .sort()
-    .join("_");
+export function getRoomID(users: string[]) {
+  return users.sort().join("_");
 }
 
 class _RoomCRUD {
@@ -72,7 +69,7 @@ class _RoomCRUD {
     room.description = room.description.trim();
     return kPrisma.room
       .upsert({
-        where: { id: room.id },
+        where: { id: room.id || k404.toString() },
         create: room,
         update: room,
       })

src/services/db/user.ts

@@ -1,5 +1,5 @@
 import { Prisma, User } from "@prisma/client";
-import { kPrisma } from ".";
+import { k404, kPrisma } from ".";
 
 class _UserCRUD {
   async count() {
@@ -50,7 +50,7 @@ class _UserCRUD {
     user.profile = user.profile.trim();
     return kPrisma.user
       .upsert({
-        where: { id: user.id },
+        where: { id: user.id || k404.toString() },
         create: user,
         update: user,
       })

src/utils/diff.ts

@@ -0,0 +1,103 @@
+// Source: https://github.com/AsyncBanana/microdiff
+
+interface Difference {
+  type: "CREATE" | "REMOVE" | "CHANGE";
+  path: (string | number)[];
+  value?: any;
+}
+interface Options {
+  cyclesFix: boolean;
+}
+
+const t = true;
+const richTypes = { Date: t, RegExp: t, String: t, Number: t };
+
+export function isEqual(oldObj: any, newObj: any): boolean {
+  return (
+    diff(
+      {
+        obj: oldObj,
+      },
+      { obj: newObj }
+    ).length < 1
+  );
+}
+
+export const isNotEqual = (oldObj: any, newObj: any) =>
+  !isEqual(oldObj, newObj);
+
+export function diff(
+  obj: Record<string, any> | any[],
+  newObj: Record<string, any> | any[],
+  options: Partial<Options> = { cyclesFix: true },
+  _stack: Record<string, any>[] = []
+): Difference[] {
+  const diffs: Difference[] = [];
+  const isObjArray = Array.isArray(obj);
+
+  for (const key in obj) {
+    const objKey = (obj as any)[key];
+    const path = isObjArray ? Number(key) : key;
+    if (!(key in newObj)) {
+      diffs.push({
+        type: "REMOVE",
+        path: [path],
+      });
+      continue;
+    }
+    const newObjKey = (newObj as any)[key];
+    const areObjects =
+      typeof objKey === "object" && typeof newObjKey === "object";
+    if (
+      objKey &&
+      newObjKey &&
+      areObjects &&
+      !(richTypes as any)[Object.getPrototypeOf(objKey).constructor.name] &&
+      (options.cyclesFix ? !_stack.includes(objKey) : true)
+    ) {
+      const nestedDiffs = diff(
+        objKey,
+        newObjKey,
+        options,
+        options.cyclesFix ? _stack.concat([objKey]) : []
+      );
+      // eslint-disable-next-line prefer-spread
+      diffs.push.apply(
+        diffs,
+        nestedDiffs.map((difference) => {
+          difference.path.unshift(path);
+
+          return difference;
+        })
+      );
+    } else if (
+      objKey !== newObjKey &&
+      !(
+        areObjects &&
+        (Number.isNaN(objKey)
+          ? String(objKey) === String(newObjKey)
+          : Number(objKey) === Number(newObjKey))
+      )
+    ) {
+      diffs.push({
+        path: [path],
+        type: "CHANGE",
+        value: newObjKey,
+      });
+    }
+  }
+
+  const isNewObjArray = Array.isArray(newObj);
+
+  for (const key in newObj) {
+    if (!(key in obj)) {
+      diffs.push({
+        type: "CREATE",
+        path: [isNewObjArray ? Number(key) : key],
+        value: (newObj as any)[key],
+      });
+    }
+  }
+
+  return diffs;
+}

src/utils/type.ts

@@ -0,0 +1,3 @@
+export type DeepPartial<T> = {
+  [P in keyof T]?: T[P] extends object ? DeepPartial<T[P]> : T[P];
+};

tests/db/index.ts

@@ -0,0 +1,23 @@
+import { assert } from "console";
+import { ConversationManager } from "../../src/services/bot/conversation";
+import { println } from "../../src/utils/base";
+
+export async function testDB() {
+  const manager = new ConversationManager({
+    bot: {
+      name: "小爱同学",
+      profile: "我是小爱同学，机器人",
+    },
+    master: {
+      name: "王黎",
+      profile: "我是王黎，人类",
+    },
+    room: {
+      name: "客厅",
+      description: "王黎的客厅，小爱同学放在角落里",
+    },
+  });
+  const { room } = await manager.loadOrUpdateConfig();
+  assert(room, "❌ load config failed");
+  println("✅ hello world！");
+}

tests/index.ts

@@ -2,11 +2,13 @@ import dotenv from "dotenv";
 import { println } from "../src/utils/base";
 import { kBannerASCII } from "../src/utils/string";
 import { runWithDB } from "../src/services/db";
+import { testDB } from "./db";
 
 dotenv.config();
 
 async function main() {
   println(kBannerASCII);
+  testDB();
 }
 
 runWithDB(main);