From 413a80cbf8425c31bbf8c69265a3e8dff411e208 Mon Sep 17 00:00:00 2001 From: WJG Date: Tue, 27 Feb 2024 23:35:00 +0800 Subject: [PATCH] feat: add initDB --- package.json | 6 +- .../migration.sql | 0 prisma/schema.prisma | 2 +- src/index.ts | 4 +- src/runner.ts | 3 +- src/services/db/index.ts | 19 +++- src/services/openai.ts | 64 ++++++------ src/utils/io.ts | 97 +++++++++++++++++-- src/utils/shell.ts | 14 +++ tests/index.ts | 3 +- yarn.lock | 43 ++++++++ 11 files changed, 204 insertions(+), 51 deletions(-) rename prisma/migrations/{20240225141130_hello => 20240227153046_init}/migration.sql (100%) create mode 100644 src/utils/shell.ts diff --git a/package.json b/package.json index 3cdc243..e527e69 100644 --- a/package.json +++ b/package.json @@ -20,19 +20,21 @@ ], "scripts": { "build": "tsup", - "db:gen": "npx prisma migrate dev --name hello", + "db:gen": "npx prisma migrate dev --name init", "db:reset": "rm .bot.json && npx prisma migrate reset", - "prepublish": "yarn db:gen && npm run build" + "prepublish": "npm run db:gen && npm run build" }, "dependencies": { "@prisma/client": "^5.8.1", "axios": "^1.6.5", + "fs-extra": "^11.2.0", "https-proxy-agent": "^7.0.4", "mi-service-lite": "^2.0.0", "openai": "^4.28.0", "prisma": "^5.8.1" }, "devDependencies": { + "@types/fs-extra": "^11.0.4", "@types/node": "^20.4.9", "dotenv": "^16.3.2", "ts-node": "^10.9.2", diff --git a/prisma/migrations/20240225141130_hello/migration.sql b/prisma/migrations/20240227153046_init/migration.sql similarity index 100% rename from prisma/migrations/20240225141130_hello/migration.sql rename to prisma/migrations/20240227153046_init/migration.sql diff --git a/prisma/schema.prisma b/prisma/schema.prisma index a48dc48..300d100 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -7,7 +7,7 @@ generator client { datasource db { provider = "sqlite" - url = "file:./hello.db" + url = "file:../.mi-gpt.db" } model User { diff --git a/src/index.ts b/src/index.ts index 98a6a50..88820eb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ import { AISpeaker, AISpeakerConfig } from "./services/speaker/ai"; import { MyBot, MyBotConfig } from "./services/bot"; -import { runWithDB } from "./services/db"; +import { initDB, runWithDB } from "./services/db"; import { kBannerASCII } from "./utils/string"; export type MiGPTConfig = Omit & { @@ -35,7 +35,7 @@ export class MiGPT { } async start() { - // todo init DB + await initDB(".mi-gpt.db"); const main = () => { console.log(kBannerASCII); return this.ai.run(); diff --git a/src/runner.ts b/src/runner.ts index a59114b..53d1ab5 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -1,4 +1,3 @@ -import { runWithDB } from "./services/db"; import { println } from "./utils/base"; import { kBannerASCII } from "./utils/string"; @@ -6,4 +5,4 @@ async function main() { println(kBannerASCII); } -runWithDB(main); +main(); diff --git a/src/services/db/index.ts b/src/services/db/index.ts index 93800c9..a22c022 100644 --- a/src/services/db/index.ts +++ b/src/services/db/index.ts @@ -1,13 +1,15 @@ import { PrismaClient } from "@prisma/client"; import { Logger } from "../../utils/log"; +import { deleteFile, exists } from "../../utils/io"; +import { Shell } from "../../utils/shell"; export const k404 = -404; export const kPrisma = new PrismaClient(); -export const kDBLogger = Logger.create({ tag: "DB" }); +export const kDBLogger = Logger.create({ tag: "database" }); export function runWithDB(main: () => Promise) { - main() + return main() .then(async () => { await kPrisma.$disconnect(); }) @@ -24,3 +26,16 @@ export function getSkipWithCursor(skip: number, cursorId: any) { cursor: cursorId ? { id: cursorId } : undefined, }; } + +export async function initDB(dbPath: string) { + if (!exists(dbPath)) { + const isExternal = exists("node_modules/mi-gpt/prisma"); + const withSchema = isExternal + ? "--schema node_modules/mi-gpt/prisma/schema.prisma" + : ""; + await deleteFile(".bot.json"); + await Shell.run(`npx prisma migrate dev --name init ${withSchema}`); + } + const success = exists(dbPath); + kDBLogger.assert(success, "初始化数据库失败！"); +} diff --git a/src/services/openai.ts b/src/services/openai.ts index cd306c4..58cdae9 100644 --- a/src/services/openai.ts +++ b/src/services/openai.ts @@ -25,16 +25,22 @@ class OpenAIClient { traceOutput = true; private _logger = Logger.create({ tag: "Open AI" }); - private _client = new OpenAI({ - httpAgent: kProxyAgent, - apiKey: kEnvs.OPENAI_API_KEY!, - }); + private _client?: OpenAI; + private _init() { + if (!this._client) { + this._client = new OpenAI({ + httpAgent: kProxyAgent, + apiKey: kEnvs.OPENAI_API_KEY!, + }); + } + } private _abortCallbacks: Record = { // requestId: abortStreamCallback }; abort(requestId: string) { + this._init(); if (this._abortCallbacks[requestId]) { this._abortCallbacks[requestId](); delete this._abortCallbacks[requestId]; @@ -42,6 +48,7 @@ class OpenAIClient { } async chat(options: ChatOptions) { + this._init(); let { user, system, @@ -65,20 +72,18 @@ class OpenAIClient { this._abortCallbacks[requestId] = () => controller.abort(); signal = controller.signal; } - const chatCompletion = await this._client.chat.completions - .create( - { - model, - tools, - messages: [...systemMsg, { role: "user", content: user }], - response_format: jsonMode ? { type: "json_object" } : undefined, - }, - { signal } - ) - .catch((e) => { - this._logger.error("openai chat failed", e); - return null; - }); + const chatCompletion = await this._client!.chat.completions.create( + { + model, + tools, + messages: [...systemMsg, { role: "user", content: user }], + response_format: jsonMode ? { type: "json_object" } : undefined, + }, + { signal } + ).catch((e) => { + this._logger.error("openai chat failed", e); + return null; + }); const message = chatCompletion?.choices?.[0]?.message; if (trace && this.traceOutput) { this._logger.log(`✅ Answer: ${message?.content ?? "None"}`.trim()); @@ -91,6 +96,7 @@ class OpenAIClient { onStream?: (text: string) => void; } ) { + this._init(); let { user, system, @@ -109,18 +115,16 @@ class OpenAIClient { const systemMsg: ChatCompletionMessageParam[] = system ? [{ role: "system", content: system }] : []; - const stream = await this._client.chat.completions - .create({ - model, - tools, - stream: true, - messages: [...systemMsg, { role: "user", content: user }], - response_format: jsonMode ? { type: "json_object" } : undefined, - }) - .catch((e) => { - this._logger.error("❌ openai chat failed", e); - return null; - }); + const stream = await this._client!.chat.completions.create({ + model, + tools, + stream: true, + messages: [...systemMsg, { role: "user", content: user }], + response_format: jsonMode ? { type: "json_object" } : undefined, + }).catch((e) => { + this._logger.error("❌ openai chat failed", e); + return null; + }); if (!stream) { return; } diff --git a/src/utils/io.ts b/src/utils/io.ts index e42966e..24275c0 100644 --- a/src/utils/io.ts +++ b/src/utils/io.ts @@ -1,5 +1,5 @@ -import * as fs from "fs"; -import * as path from "path"; +import fs from "fs-extra"; +import path from "path"; import { jsonDecode, jsonEncode } from "./base"; @@ -7,14 +7,7 @@ export const kRoot = process.cwd(); export const exists = (filePath: string) => fs.existsSync(filePath); -export const deleteFile = (filePath: string) => { - try { - fs.rmSync(filePath); - return true; - } catch { - return false; - } -}; +export const getFullPath = (filePath: string) => path.resolve(filePath); export const getFiles = (dir: string) => { return new Promise((resolve) => { @@ -89,3 +82,87 @@ export const readJSONSync = (filePath: string) => export const writeJSON = (filePath: string, content: any) => writeFile(filePath, jsonEncode(content) ?? "", "utf8"); + +export const deleteFile = (filePath: string) => { + try { + fs.rmSync(filePath); + return true; + } catch { + return false; + } +}; + +export const copyFile = ( + from: string, + to: string, + mode?: number | undefined +) => { + if (!fs.existsSync(from)) { + return false; + } + const dirname = path.dirname(to); + if (!fs.existsSync(dirname)) { + fs.mkdirSync(dirname, { recursive: true }); + } + return new Promise((resolve) => { + const callback = (err: any) => { + resolve(err ? false : true); + }; + if (mode) { + fs.copyFile(from, to, mode, callback); + } else { + fs.copyFile(from, to, callback); + } + }); +}; + +export const copyFileSync = ( + from: string, + to: string, + mode?: number | undefined +) => { + if (!fs.existsSync(from)) { + return false; + } + const dirname = path.dirname(to); + if (!fs.existsSync(dirname)) { + fs.mkdirSync(dirname, { recursive: true }); + } + try { + fs.copyFileSync(from, to, mode); + return true; + } catch { + return false; + } +}; + +export const moveFile = (from: string, to: string) => { + if (!fs.existsSync(from)) { + return false; + } + const dirname = path.dirname(to); + if (!fs.existsSync(dirname)) { + fs.mkdirSync(dirname, { recursive: true }); + } + return new Promise((resolve) => { + fs.rename(from, to, (err) => { + resolve(err ? false : true); + }); + }); +}; + +export const moveFileSync = (from: string, to: string) => { + if (!fs.existsSync(from)) { + return false; + } + const dirname = path.dirname(to); + if (!fs.existsSync(dirname)) { + fs.mkdirSync(dirname, { recursive: true }); + } + try { + fs.renameSync(from, to); + return true; + } catch { + return false; + } +}; diff --git a/src/utils/shell.ts b/src/utils/shell.ts new file mode 100644 index 0000000..a25638c --- /dev/null +++ b/src/utils/shell.ts @@ -0,0 +1,14 @@ +import { exec as execSync } from "child_process"; +import { promisify } from "util"; + +const exec = promisify(execSync); + +export class Shell { + static async run(command: string) { + return exec(command); + } + + static get args() { + return process.argv.slice(2); + } +} diff --git a/tests/index.ts b/tests/index.ts index 12a3cea..a4deae6 100644 --- a/tests/index.ts +++ b/tests/index.ts @@ -1,7 +1,6 @@ import dotenv from "dotenv"; import { println } from "../src/utils/base"; import { kBannerASCII } from "../src/utils/string"; -import { runWithDB } from "../src/services/db"; import { testDB } from "./db"; import { testSpeaker } from "./speaker"; import { testOpenAI } from "./openai"; @@ -21,4 +20,4 @@ async function main() { testMiGPT(); } -runWithDB(main); +main(); diff --git a/yarn.lock b/yarn.lock index c1fbcdf..b10bab2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -333,6 +333,21 @@ resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== +"@types/fs-extra@^11.0.4": + version "11.0.4" + resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-11.0.4.tgz#e16a863bb8843fba8c5004362b5a73e17becca45" + integrity sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ== + dependencies: + "@types/jsonfile" "*" + "@types/node" "*" + +"@types/jsonfile@*": + version "6.1.4" + resolved "https://registry.yarnpkg.com/@types/jsonfile/-/jsonfile-6.1.4.tgz#614afec1a1164e7d670b4a7ad64df3e7beb7b702" + integrity sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ== + dependencies: + "@types/node" "*" + "@types/node-fetch@^2.6.4": version "2.6.11" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" @@ -717,6 +732,15 @@ formdata-node@^4.3.2: node-domexception "1.0.0" web-streams-polyfill "4.0.0-beta.3" +fs-extra@^11.2.0: + version "11.2.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.2.0.tgz#e70e17dfad64232287d01929399e0ea7c86b0e5b" + integrity sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + fsevents@~2.3.2: version "2.3.3" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" @@ -757,6 +781,11 @@ globby@^11.0.3: merge2 "^1.4.1" slash "^3.0.0" +graceful-fs@^4.1.6, graceful-fs@^4.2.0: + version "4.2.11" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" + integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== + https-proxy-agent@^7.0.4: version "7.0.4" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz#8e97b841a029ad8ddc8731f26595bad868cb4168" @@ -840,6 +869,15 @@ joycon@^3.0.1: resolved "https://registry.yarnpkg.com/joycon/-/joycon-3.1.1.tgz#bce8596d6ae808f8b68168f5fc69280996894f03" integrity sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw== +jsonfile@^6.0.1: + version "6.1.0" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" + integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ== + dependencies: + universalify "^2.0.0" + optionalDependencies: + graceful-fs "^4.1.6" + lilconfig@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-3.0.0.tgz#f8067feb033b5b74dab4602a5f5029420be749bc" @@ -1291,6 +1329,11 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +universalify@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.1.tgz#168efc2180964e6386d061e094df61afe239b18d" + integrity sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw== + v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"