refactor: 重构数据库初始化与迁移逻辑

README.md

@@ -7,65 +7,23 @@
 
 ## 数据库准备
 
-本服务的部分模块依赖外部数据源 `jianshu` 和 `jpep` 数据库，您需事先下载并进行导入。
+本服务的部分模块依赖外部数据源 `jianshu` 与 `jpep` 数据库，您需事先下载并进行导入。
 
-创建用户：
+进入 `sql` 目录：
 
-```sql
-CREATE ROLE jtools LOGIN PASSWORD 'jtools';
-```
-
-创建数据库：
-
-```sql
-CREATE DATABASE jtools WITH OWNER = jtools;
-CREATE DATABASE logs;
-```
-
-创建扩展：
-
-（`jianshu` 数据库）
-
-```sql
-CREATE EXTENSION IF NOT EXISTS pg_trgm;
-```
-
-为用户授权：
-
-（`logs` 数据库）
-
-```sql
-GRANT CREATE ON SCHEMA public TO jtools;
-```
-
-（`jianshu` 数据库）
-
-```sql
-GRANT SELECT ON TABLE article_earning_ranking_records TO jtools;
-GRANT SELECT ON TABLE lottery_win_records TO jtools;
-GRANT SELECT ON TABLE users TO jtools;
-```
-
-（`jpep` 数据库）
-
-```sql
-GRANT SELECT ON TABLE ftn_macket_records TO jtools;
-GRANT SELECT ON TABLE ftn_orders TO jtools;
+```shell
+cd sql
 ```
 
-创建索引：
+如果您需要修改数据库用户名和密码，请修改 `sql` 目录下的 `0.sql` 和每个子目录下的 `0.sql` 文件。
 
-（`jianshu` 数据库）
+您需要一个具有创建用户和数据库权限的用户（一般是超级用户）来完成数据库准备。
 
-```sql
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_article_earning_ranking_records_ranking ON article_earning_ranking_records (ranking);
+每个目录中的 SQL 脚本均应按照编号顺序执行。
 
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_lottery_win_records_time ON lottery_win_records (time);
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_lottery_win_records_user_slug ON lottery_win_records (user_slug);
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_lottery_win_records_award_name ON lottery_win_records (award_name);
+首先，执行 `sql` 目录下的脚本。
 
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_users_name ON users USING gin (name gin_trgm_ops);
-```
+依次切换到与 `sql` 下的子目录（数据库目录）名称相同的数据库中，先执行每个表目录中的 SQL 脚本，再执行数据库目录下的 SQL 脚本。
 
 ## 配置
 
@@ -82,7 +40,7 @@ cp config.example.toml config.toml
 - jpep_postgres.host 填写 `postgres`
 - uvicorn.host 填写 `0.0.0.0`
 
-同时，您需要填写正确的 `postgres.user` 和 `postgres.password`。
+同时，您需要填写正确的 `{db_name}_postgres.user` 和 `{db_name}_postgres.password`。
 
 `word_split_access_key` 为具有 [NLP 服务](https://ai.aliyun.com/nlp) 使用权限的阿里云用户 Access Key，您可在 [RAM 访问控制](https://ram.console.aliyun.com) 中创建用户，并为其赋予 `AliyunNLPReadOnlyAccess` 权限。
 

backend/models/debug_project_record.py

@@ -17,24 +17,6 @@ class DebugProjectRecord(Table, frozen=True):
     user_slug: NonEmptyStr
     reward: PositiveInt
 
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jtools_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS debug_project_records (
-                    id SMALLSERIAL CONSTRAINT pk_debug_project_records_id PRIMARY KEY,
-                    date DATE NOT NULL,
-                    type TEXT NOT NULL,
-                    module TEXT NOT NULL,
-                    description TEXT NOT NULL,
-                    user_name TEXT NOT NULL,
-                    user_slug VARCHAR(12) NOT NULL,
-                    reward SMALLINT NOT NULL
-                );
-                """
-            )
-
     @classmethod
     async def iter(cls) -> AsyncGenerator["DebugProjectRecord", None]:
         async with jtools_pool.get_conn() as conn:

backend/models/tech_stack.py

@@ -2,7 +2,7 @@
 from enum import Enum
 from typing import Optional
 
-from sshared.postgres import Table, create_enum
+from sshared.postgres import Table
 from sshared.strict_struct import NonEmptyStr
 
 from utils.db import jtools_pool
@@ -27,32 +27,6 @@ class TechStack(Table, frozen=True):
     description: NonEmptyStr
     url: NonEmptyStr
 
-    @classmethod
-    async def _create_enum(cls) -> None:
-        async with jtools_pool.get_conn() as conn:
-            await create_enum(
-                conn=conn, name="enum_tech_stacks_type", enum_class=TypeEnum
-            )
-            await create_enum(
-                conn=conn, name="enum_tech_stacks_scope", enum_class=ScopeEnum
-            )
-
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jtools_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS tech_stacks (
-                    name TEXT NOT NULL CONSTRAINT pk_tech_stacks_name PRIMARY KEY,
-                    type enum_tech_stacks_type NOT NULL,
-                    scope enum_tech_stacks_scope NOT NULL,
-                    is_self_developed BOOLEAN NOT NULL,
-                    description TEXT NOT NULL,
-                    url TEXT NOT NULL
-                );
-                """
-            )
-
     @classmethod
     async def iter(
         cls, scope: Optional[ScopeEnum] = None

backend/models/tool.py

@@ -2,7 +2,7 @@
 from typing import Optional
 
 from psycopg.types.json import Jsonb
-from sshared.postgres import Table, create_enum
+from sshared.postgres import Table
 from sshared.strict_struct import NonEmptyStr
 
 from utils.db import jtools_pool
@@ -26,32 +26,6 @@ class Tool(Table, frozen=True):
     data_count_table: Optional[NonEmptyStr]
     data_source: Optional[dict[str, str]]
 
-    @classmethod
-    async def _create_enum(cls) -> None:
-        async with jtools_pool.get_conn() as conn:
-            await create_enum(
-                conn=conn, name="enum_tools_status", enum_class=StatusEnum
-            )
-
-    @classmethod
-    async def _create_table(cls) -> None:
-        async with jtools_pool.get_conn() as conn:
-            await conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS tools (
-                    slug TEXT CONSTRAINT pk_tools_slug PRIMARY KEY,
-                    status enum_tools_status NOT NULL,
-                    status_description TEXT,
-                    data_update_freq TEXT NOT NULL,
-                    last_update_time_table TEXT,
-                    last_update_time_order_by TEXT,
-                    last_update_time_target_field TEXT,
-                    data_count_table TEXT,
-                    data_source JSONB
-                );
-                """
-            )
-
     @classmethod
     async def init(cls) -> None:
         await super().init()

backend/sql/0.sql

@@ -0,0 +1,10 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+CREATE ROLE jtools LOGIN PASSWORD 'jtools';
+
+CREATE DATABASE jtools WITH OWNER = jtools;
+
+GRANT CONNECT ON DATABASE jtools TO jtools;
+GRANT CONNECT ON DATABASE jianshu TO jtools;
+GRANT CONNECT ON DATABASE jpep TO jtools;

backend/sql/jianshu/0.sql

@@ -0,0 +1,14 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+GRANT SELECT ON TABLE article_earning_ranking_records TO jtools;
+GRANT SELECT ON TABLE lottery_win_records TO jtools;
+GRANT SELECT ON TABLE users TO jtools;
+
+CREATE EXTENSION pg_trgm;
+
+CREATE INDEX CONCURRENTLY idx_article_earning_ranking_records_ranking ON article_earning_ranking_records (ranking);
+CREATE INDEX CONCURRENTLY idx_lottery_win_records_time ON lottery_win_records (time);
+CREATE INDEX CONCURRENTLY idx_lottery_win_records_user_slug ON lottery_win_records (user_slug);
+CREATE INDEX CONCURRENTLY idx_lottery_win_records_award_name ON lottery_win_records (award_name);
+CREATE INDEX CONCURRENTLY idx_users_name ON users USING gin (name gin_trgm_ops);

backend/sql/jpep/0.sql

@@ -0,0 +1,5 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+GRANT SELECT ON TABLE ftn_macket_records TO jtools;
+GRANT SELECT ON TABLE ftn_orders TO jtools;

backend/sql/jtools/0.sql

@@ -0,0 +1,6 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+GRANT SELECT ON TABLE debug_project_records TO jtools;
+GRANT SELECT ON TABLE tech_stacks TO jtools;
+GRANT SELECT, INSERT ON TABLE tools TO jtools;

backend/sql/jtools/debug_project_records/0.sql

@@ -0,0 +1,13 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+CREATE TABLE debug_project_records (
+    id SMALLSERIAL CONSTRAINT pk_debug_project_records_id PRIMARY KEY,
+    date DATE NOT NULL,
+    type TEXT NOT NULL,
+    module TEXT NOT NULL,
+    description TEXT NOT NULL,
+    user_name TEXT NOT NULL,
+    user_slug VARCHAR(12) NOT NULL,
+    reward SMALLINT NOT NULL
+);

backend/sql/jtools/tech_stacks/0.sql

@@ -0,0 +1,14 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+CREATE TYPE enum_tech_stacks_type AS ENUM ('LIBRARY', 'EXTERNAL_SERVICE');
+CREATE TYPE enum_tech_stacks_scope AS ENUM ('FRONTEND', 'BACKEND', 'TOOLCHAIN');
+
+CREATE TABLE tech_stacks (
+    name TEXT NOT NULL CONSTRAINT pk_tech_stacks_name PRIMARY KEY,
+    type enum_tech_stacks_type NOT NULL,
+    scope enum_tech_stacks_scope NOT NULL,
+    is_self_developed BOOLEAN NOT NULL,
+    description TEXT NOT NULL,
+    url TEXT NOT NULL
+);

backend/sql/jtools/tools/0.sql

@@ -0,0 +1,16 @@
+-- date: 2024-11-13
+-- description: 初始化
+
+CREATE TYPE enum_tools_status AS ENUM ('NORMAL', 'DOWNGRADED', 'UNAVAILABLE');
+
+CREATE TABLE tools (
+    slug TEXT CONSTRAINT pk_tools_slug PRIMARY KEY,
+    status enum_tools_status NOT NULL,
+    status_description TEXT,
+    data_update_freq TEXT NOT NULL,
+    last_update_time_table TEXT,
+    last_update_time_order_by TEXT,
+    last_update_time_target_field TEXT,
+    data_count_table TEXT,
+    data_source JSONB
+);