diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000..30c082949f Binary files /dev/null and b/.DS_Store differ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 201abc7c22..9ed9ea433e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,9 +11,9 @@ on: workflow_dispatch: inputs: debug_enabled: - description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' + description: "Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)" required: false - default: 'false' + default: "false" jobs: test: @@ -48,8 +48,10 @@ jobs: if: steps.cache.outputs.cache-hit != 'true' run: | python -m pip install --upgrade pip + python -m pip install types-setuptools python -m pip install "poetry" python -m poetry self add poetry-version-plugin + - name: Configure poetry run: python -m poetry config virtualenvs.create false - name: Install Dependencies @@ -78,7 +80,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: "3.8" - name: Get coverage files uses: actions/download-artifact@v3 @@ -100,7 +102,7 @@ jobs: path: htmlcov # https://github.com/marketplace/actions/alls-green#why - alls-green: # This job does nothing and is only used for the branch protection + alls-green: # This job does nothing and is only used for the branch protection if: always() needs: - coverage-combine diff --git a/.gitignore b/.gitignore index 4006069389..5fc9253dc1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ coverage.xml site *.db .cache +!docs_src/**/env.py diff --git a/docs/advanced/index.md b/docs/advanced/index.md index f6178249ce..baa50cc37b 100644 --- a/docs/advanced/index.md +++ b/docs/advanced/index.md @@ -5,6 +5,5 @@ The **Advanced User Guide** is gradually growing, you can already read about som At some point it will include: * How to use `async` and `await` with the async session. -* How to run migrations. * How to combine **SQLModel** models with SQLAlchemy. * ...and more. πŸ€“ diff --git a/docs/advanced/migrations.md b/docs/advanced/migrations.md new file mode 100644 index 0000000000..afd83ce876 --- /dev/null +++ b/docs/advanced/migrations.md @@ -0,0 +1,211 @@ +# Manage migrations + +SQLModel integrates [Alembic](https://alembic.sqlalchemy.org/en/latest/) to manage migrations and DB Schema. + + +## **SQLModel** Code - Models and Migrations + +Now let's start with the SQLModel code. + +We will start with the **simplest version**, with just heroes (no teams yet). + +This is almost the same code as you start to know by heart: + +```Python +{!./docs_src/tutorial/migrations/simple_hero_migration/models.py!} +``` + +Let's jump in your shell and init migrations: + +
+ +```console +$ sqlmodel migrations init +Creating directory '/path/to/your/project/migrations' ... done +Creating directory '/path/to/your/project/migrations/versions' ... done +Generating /path/to/your/project/migrations/script.py.mako ... done +Generating /path/to/your/project/migrations/env.py ... done +Generating /path/to/your/project/migrations/README ... done +Generating /path/to/your/project/alembic.ini ... done +Adding '/path/to/your/project/migrations/__init__.py' ... done +Adding '/path/to/your/project/migrations/versions/__init__.py' ... done +Please edit configuration/connection/logging settings in '/path/to/your/project/alembic.ini' before proceeding. +``` +
+ +Few things happended under the hood. + +Let's review what happened: Below files just got created! + +```hl_lines="5-12" +. +β”œβ”€β”€ project + β”œβ”€β”€ __init__.py + β”œβ”€β”€ models.py + β”œβ”€β”€ alembic.ini + └── migrations + β”œβ”€β”€ __init__.py + β”œβ”€β”€ env.py + β”œβ”€β”€ README + β”œβ”€β”€ script.py.mako + └── versions + └── __init__.py +``` + +Let's review them step by step. + +## Alembic configuration + +**`alembic.ini`** gives all the details of Alembic's configuration. You shouldn't \*have to\* touch that a lot, but for our setup, we'll need to change few things. + +We need to tell alembic how to connect to the database: + +```ini hl_lines="10" +{!./docs_src/tutorial/migrations/simple_hero_migration/alembic001.ini[ln:1-5]!} + +#.... Lot's of configuration! + + +{!./docs_src/tutorial/migrations/simple_hero_migration/alembic001.ini[ln:63]!} # πŸ‘ˆ Let's Change that! +``` +Adapting our file, you will have: + +```ini hl_lines="10" +{!./docs_src/tutorial/migrations/simple_hero_migration/alembic.ini[ln:1-5]!} + +#.... Lot's of configuration! + + +{!./docs_src/tutorial/migrations/simple_hero_migration/alembic.ini[ln:63]!} # πŸ‘ˆ To that +``` + +For the full document, refer to [Alembic's official documentation](https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file) + +**`./migrations/env.py`** is another file we'll need to configure: +It gives which Tables you want to migrate, let's open it and: + +1. Import our models +2. Change `target_metadata` value + +```Python hl_lines="5 7" +{!./docs_src/tutorial/migrations/simple_hero_migration/migrations/001.py[ln:1-5]!} πŸ‘ˆ Import your model +# ..... +{!./docs_src/tutorial/migrations/simple_hero_migration/migrations/001.py[ln:19]!} πŸ‘ˆ Set you Metadata value +``` + +## Create an apply your first migration +!!! success + πŸ‘πŸŽ‰At this point, you are ready to track your DB Schema ! + +Let's create you first migration + +
+```console +$ sqlmodel migrations init +INFO [alembic.runtime.migration] Context impl SQLiteImpl. +INFO [alembic.runtime.migration] Will assume non-transactional DDL. +INFO [alembic.autogenerate.compare] Detected added table 'hero' + Generating /path/to/your/project/migrations/versions/0610946706a0_.py ... done + +``` + +
+ +Alembic did its magic and started to track your `Hero` model! +It created a new file `0610946706a0_.py` + + +```hl_lines="13" +. +└── project + β”œβ”€β”€ __init__.py + β”œβ”€β”€ models.py + β”œβ”€β”€ alembic.ini + └── migrations + β”œβ”€β”€ __init__.py + β”œβ”€β”€ env.py + β”œβ”€β”€ README + β”œβ”€β”€ script.py.mako + └── versions + β”œβ”€β”€ __init__.py + └── 0610946706a0_.py +``` + +Let's prepare for our migration, and see what will happen. + +
+``` +$ sqlmodel migrations show +Rev: 50624637e300 (head) +Parent: +Path: /path/to/your/project/migrations/versions/0610946706a0_.py #πŸ‘ˆ That's our file + + empty message + + Revision ID: 50624637e300 + Revises: + Create Date: 2023-10-31 19:40:22.084162 +``` +
+ +We are pretty sure about what will happen during migration, let's do it: + +
+``` +$ sqlmodel migrations upgrade +INFO [alembic.runtime.migration] Context impl SQLiteImpl. +INFO [alembic.runtime.migration] Will assume non-transactional DDL. +INFO [alembic.runtime.migration] Running upgrade -> 1e606859995a, migrating me iam famous +``` +
+ + +Let's open our DB browser and check it out: + + + + + +## Change you versions file name + +Why the heck `0610946706a0_.py`?!!!! + +The goal is to have a unique revision name to avoid collision. +In order to have a cleaner file name, we can edit `alembic.ini` and uncomment + +```ini +{!./docs_src/tutorial/migrations/simple_hero_migration/alembic.ini[ln:11]!} #πŸ‘ˆ Uncoment this line +``` + +Let's remove `0610946706a0_.py` and start it over. + +
+```console +$ sqlmodel migrations revision +INFO [alembic.runtime.migration] Context impl SQLiteImpl. +INFO [alembic.runtime.migration] Will assume non-transactional DDL. +INFO [alembic.autogenerate.compare] Detected added table 'hero' + Generating /path/to/your/project/migrations/versions//2023_10_31_1940-50624637e300_.py ... done +``` +
+ +Much better, not perfect but better. + +To get more details just by looking at you file name, you can also run + + +
+```console +$ sqlmodel migrations revision "migrate me iam famous" +INFO [alembic.runtime.migration] Context impl SQLiteImpl. +INFO [alembic.runtime.migration] Will assume non-transactional DDL. +INFO [alembic.autogenerate.compare] Detected added table 'hero' + Generating /path/to/your/project/migrations/versions/2023_10_31_1946-1e606859995a_migrate_me_iam_famous.py + ... done +``` +
+ + +You can think of "migrate me iam famous" as a message you add to you migration. + +It helps you keep track of what they do, pretty much like in `git` diff --git a/docs_src/tutorial/migrations/simple_hero_migration/__init__.py b/docs_src/tutorial/migrations/simple_hero_migration/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs_src/tutorial/migrations/simple_hero_migration/alembic.ini b/docs_src/tutorial/migrations/simple_hero_migration/alembic.ini new file mode 100644 index 0000000000..f9363281d5 --- /dev/null +++ b/docs_src/tutorial/migrations/simple_hero_migration/alembic.ini @@ -0,0 +1,116 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = migrations + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = sqlite:///database.db + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/docs_src/tutorial/migrations/simple_hero_migration/alembic001.ini b/docs_src/tutorial/migrations/simple_hero_migration/alembic001.ini new file mode 100644 index 0000000000..07489da977 --- /dev/null +++ b/docs_src/tutorial/migrations/simple_hero_migration/alembic001.ini @@ -0,0 +1,116 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = migrations + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/docs_src/tutorial/migrations/simple_hero_migration/migrations/001.py b/docs_src/tutorial/migrations/simple_hero_migration/migrations/001.py new file mode 100644 index 0000000000..c865fe6961 --- /dev/null +++ b/docs_src/tutorial/migrations/simple_hero_migration/migrations/001.py @@ -0,0 +1,75 @@ +from logging.config import fileConfig + +from alembic import context +from models import Hero +from sqlalchemy import engine_from_config, pool + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Hero.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/docs_src/tutorial/migrations/simple_hero_migration/migrations/README b/docs_src/tutorial/migrations/simple_hero_migration/migrations/README new file mode 100644 index 0000000000..45acd9d199 --- /dev/null +++ b/docs_src/tutorial/migrations/simple_hero_migration/migrations/README @@ -0,0 +1,2 @@ +Generic single-database configuration. +I'm the best! diff --git a/docs_src/tutorial/migrations/simple_hero_migration/migrations/__init__.py b/docs_src/tutorial/migrations/simple_hero_migration/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs_src/tutorial/migrations/simple_hero_migration/migrations/env.py b/docs_src/tutorial/migrations/simple_hero_migration/migrations/env.py new file mode 100644 index 0000000000..c865fe6961 --- /dev/null +++ b/docs_src/tutorial/migrations/simple_hero_migration/migrations/env.py @@ -0,0 +1,75 @@ +from logging.config import fileConfig + +from alembic import context +from models import Hero +from sqlalchemy import engine_from_config, pool + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Hero.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/docs_src/tutorial/migrations/simple_hero_migration/migrations/script.py.mako b/docs_src/tutorial/migrations/simple_hero_migration/migrations/script.py.mako new file mode 100644 index 0000000000..6ce3351093 --- /dev/null +++ b/docs_src/tutorial/migrations/simple_hero_migration/migrations/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/docs_src/tutorial/migrations/simple_hero_migration/migrations/versions/2023_10_31_1946-1e606859995a_migrating_me_iam_famous.py b/docs_src/tutorial/migrations/simple_hero_migration/migrations/versions/2023_10_31_1946-1e606859995a_migrating_me_iam_famous.py new file mode 100644 index 0000000000..af3f7bad85 --- /dev/null +++ b/docs_src/tutorial/migrations/simple_hero_migration/migrations/versions/2023_10_31_1946-1e606859995a_migrating_me_iam_famous.py @@ -0,0 +1,37 @@ +"""migrating me iam famous + +Revision ID: 1e606859995a +Revises: +Create Date: 2023-10-31 19:46:27.802669 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +import sqlmodel +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "1e606859995a" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "hero", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("secret_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("age", sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("hero") + # ### end Alembic commands ### diff --git a/docs_src/tutorial/migrations/simple_hero_migration/migrations/versions/__init__.py b/docs_src/tutorial/migrations/simple_hero_migration/migrations/versions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs_src/tutorial/migrations/simple_hero_migration/models.py b/docs_src/tutorial/migrations/simple_hero_migration/models.py new file mode 100644 index 0000000000..12132bb7b9 --- /dev/null +++ b/docs_src/tutorial/migrations/simple_hero_migration/models.py @@ -0,0 +1,10 @@ +from typing import Optional + +from sqlmodel import Field, SQLModel + + +class Hero(SQLModel, table=True): + id: Optional[int] = Field(default=None, primary_key=True) + name: str + secret_name: str + age: Optional[int] = None diff --git a/mkdocs.yml b/mkdocs.yml index 646af7c39e..3c82dabdd4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -85,6 +85,7 @@ nav: - Advanced User Guide: - advanced/index.md - advanced/decimal.md + - advanced/migrations.md - alternatives.md - help.md - contributing.md diff --git a/pyproject.toml b/pyproject.toml index 23fa79bf31..fb8270ad95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,16 @@ classifiers = [ python = "^3.7" SQLAlchemy = ">=1.4.36,<2.0.0" pydantic = "^1.9.0" -sqlalchemy2-stubs = {version = "*", allow-prereleases = true} +sqlalchemy2-stubs = { version = "*", allow-prereleases = true } +typer = "^0.9.0" +alembic = "^1.12.0" + +[tool.poetry.scripts] +sqlmodel = "sqlmodel.cli:cli" + +[tool.poetry.plugins."sqlmodel"] +migrations = "sqlmodel.cli.migrations.cli:migrations" + [tool.poetry.group.dev.dependencies] pytest = "^7.0.1" @@ -44,7 +53,7 @@ mkdocs-material = "9.2.7" pillow = "^9.3.0" cairosvg = "^2.5.2" mdx-include = "^1.4.1" -coverage = {extras = ["toml"], version = ">=6.2,<8.0"} +coverage = { extras = ["toml"], version = ">=6.2,<8.0" } fastapi = "^0.68.1" requests = "^2.26.0" ruff = "^0.1.2" @@ -58,11 +67,7 @@ source = "init" [tool.coverage.run] parallel = true -source = [ - "docs_src", - "tests", - "sqlmodel" -] +source = ["docs_src", "tests", "sqlmodel"] context = '${CONTEXT}' [tool.coverage.report] @@ -91,9 +96,9 @@ select = [ "UP", # pyupgrade ] ignore = [ - "E501", # line too long, handled by black - "B008", # do not perform function calls in argument defaults - "C901", # too complex + "E501", # line too long, handled by black + "B008", # do not perform function calls in argument defaults + "C901", # too complex "W191", # indentation contains tabs ] diff --git a/sqlmodel/cli/__init__.py b/sqlmodel/cli/__init__.py new file mode 100644 index 0000000000..764b7cda9f --- /dev/null +++ b/sqlmodel/cli/__init__.py @@ -0,0 +1,17 @@ +""" +An extendable and simple CLI. +Load plugins from "sqlmodel" entry points. +""" +import pkg_resources +from typer import Typer + +cli = Typer() + + +def get_entry_points(plugin_name: str = "sqlmodel", app: Typer = cli) -> None: + for entry_point in pkg_resources.iter_entry_points(plugin_name): + plugin = entry_point.load() + cli.add_typer(plugin, name=plugin.info.name) + + +get_entry_points() diff --git a/sqlmodel/cli/migrations/__init__.py b/sqlmodel/cli/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sqlmodel/cli/migrations/cli.py b/sqlmodel/cli/migrations/cli.py new file mode 100644 index 0000000000..b38ab08f75 --- /dev/null +++ b/sqlmodel/cli/migrations/cli.py @@ -0,0 +1,134 @@ +import configparser +import logging +from pathlib import Path +from typing import List, Optional + +import typer +from alembic import command +from alembic.config import Config +from typer import Typer +from typing_extensions import Annotated + +logger = logging.getLogger(__name__) +migrations = Typer( + name="migrations", help="Commands to interact with Alembic migrations" +) + + +@migrations.command() +def init( + module: Path = Path("."), + config_file: Path = Path("alembic.ini"), + directory: Path = Path("migrations"), + template: str = "generic", # Should be Literal["generic", "multidb", "async"] + target_metadata: Annotated[Optional[str], typer.Argument()] = None, +) -> None: + """Initialize Alembic""" + _config_file = module / config_file + _directory = module / directory + + config = Config(_config_file) + + template_path = Path(__file__).parent.absolute() / "templates" / template + + command.init( + config, + str(_directory), + template=str(template_path), + package=True, + ) + logger.debug("Inited alembic") + if target_metadata: + # Add target_metadata to alembic.ini + + updated_config = configparser.ConfigParser() + updated_config.read(_config_file) + updated_config["alembic"].update({"target_metadata": target_metadata}) + with open(_config_file, "w") as configfile: + updated_config.write(configfile) + logger.debug("Alembic defaults overrwritten") + + +@migrations.command() +def revision( + module: Path = Path("."), + config_file: Path = Path("alembic.ini"), + message: Annotated[Optional[str], typer.Argument()] = None, + autogenerate: bool = True, + sql: bool = False, + head: str = "head", + splice: bool = False, + # branch_label: Annotated[Optional[_RevIdType], typer.Argument()] = None, + version_path: Annotated[Optional[str], typer.Argument()] = None, + rev_id: Annotated[Optional[str], typer.Argument()] = None, + depends_on: Annotated[Optional[str], typer.Argument()] = None, +) -> None: + """Create a new Alembic revision""" + config = Config(module / config_file) + command.revision( + config, + message=message, + autogenerate=autogenerate, + sql=sql, + head=head, + splice=splice, + # branch_label=branch_label, + version_path=version_path, + rev_id=rev_id, + depends_on=depends_on, + ) + + +@migrations.command() +def show( + module: Path = Path("."), config_file: Path = Path("alembic.ini"), rev: str = "head" +) -> None: + """Show the revision""" + config = Config(module / config_file) + # Untyped function in Alembic + command.show(config, rev) # type: ignore + + +def merge( + revisions: List[str], + module: Path = Path("."), + config_file: Path = Path("alembic.ini"), + message: Annotated[Optional[str], typer.Argument()] = None, + branch_label: Annotated[Optional[List[str]], typer.Argument()] = None, + rev_id: Annotated[Optional[str], typer.Argument()] = None, +) -> None: + """Merge two revisions together, creating a new migration file""" + config = Config(module / config_file) + command.merge( + config, + revisions, + message=message, + branch_label=branch_label, + rev_id=rev_id, + ) + + +@migrations.command() +def upgrade( + revision: str = "head", + module: Path = Path("."), + config_file: Path = Path("alembic.ini"), + sql: bool = False, + tag: Annotated[Optional[str], typer.Argument()] = None, +) -> None: + """Upgrade to the given revision""" + config = Config(module / config_file) + command.upgrade(config, revision) + + +@migrations.command() +def downgrade( + revision: str = "head", + module: Path = Path("."), + config_file: Path = Path("alembic.ini"), + sql: bool = False, + tag: Annotated[Optional[str], typer.Argument()] = None, +) -> None: + """Downgrade to the given revision""" + config = Config(module / config_file) + command.downgrade(config, revision) diff --git a/sqlmodel/cli/migrations/templates/LICENSE.md b/sqlmodel/cli/migrations/templates/LICENSE.md new file mode 100644 index 0000000000..9fc0da344a --- /dev/null +++ b/sqlmodel/cli/migrations/templates/LICENSE.md @@ -0,0 +1 @@ +This folder is adapted from [Alembic](https://github.com/sqlalchemy/alembic) diff --git a/sqlmodel/cli/migrations/templates/async/README b/sqlmodel/cli/migrations/templates/async/README new file mode 100644 index 0000000000..a23d4fb519 --- /dev/null +++ b/sqlmodel/cli/migrations/templates/async/README @@ -0,0 +1 @@ +Generic single-database configuration with an async dbapi. diff --git a/sqlmodel/cli/migrations/templates/async/alembic.ini.mako b/sqlmodel/cli/migrations/templates/async/alembic.ini.mako new file mode 100644 index 0000000000..bc9f2d50ff --- /dev/null +++ b/sqlmodel/cli/migrations/templates/async/alembic.ini.mako @@ -0,0 +1,114 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = ${script_location} + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to ${script_location}/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:${script_location}/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/sqlmodel/cli/migrations/templates/async/script.py.mako b/sqlmodel/cli/migrations/templates/async/script.py.mako new file mode 100644 index 0000000000..6ce3351093 --- /dev/null +++ b/sqlmodel/cli/migrations/templates/async/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/sqlmodel/cli/migrations/templates/generic/README b/sqlmodel/cli/migrations/templates/generic/README new file mode 100644 index 0000000000..2500aa1bcf --- /dev/null +++ b/sqlmodel/cli/migrations/templates/generic/README @@ -0,0 +1 @@ +Generic single-database configuration. diff --git a/sqlmodel/cli/migrations/templates/generic/alembic.ini.mako b/sqlmodel/cli/migrations/templates/generic/alembic.ini.mako new file mode 100644 index 0000000000..c18ddb4e04 --- /dev/null +++ b/sqlmodel/cli/migrations/templates/generic/alembic.ini.mako @@ -0,0 +1,116 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = ${script_location} + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to ${script_location}/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:${script_location}/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/sqlmodel/cli/migrations/templates/generic/script.py.mako b/sqlmodel/cli/migrations/templates/generic/script.py.mako new file mode 100644 index 0000000000..6ce3351093 --- /dev/null +++ b/sqlmodel/cli/migrations/templates/generic/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/tests/test_cli/__init__.py b/tests/test_cli/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_cli/conftest.py b/tests/test_cli/conftest.py new file mode 100644 index 0000000000..55d0fb21aa --- /dev/null +++ b/tests/test_cli/conftest.py @@ -0,0 +1,25 @@ +import pkg_resources +import pytest +import typer +from typer.testing import CliRunner + +app = typer.Typer(name="dummy", help="Dummy app") + + +@pytest.fixture +def runner() -> CliRunner: + yield CliRunner() + + +@pytest.fixture +def fake_cli() -> typer.Typer: + entry_point = pkg_resources.EntryPoint( + name="dummy", module_name="tests.test_cli.conftest", attrs=["app"] + ) + entry_point.extras = [] + distribution = pkg_resources.Distribution() + entry_point.dist = distribution + distribution._ep_map = {"sqlmodel": {"dummy": entry_point}} + pkg_resources.working_set.add(distribution, "dummy") + + return app diff --git a/tests/test_cli/test_loader.py b/tests/test_cli/test_loader.py new file mode 100644 index 0000000000..1958695669 --- /dev/null +++ b/tests/test_cli/test_loader.py @@ -0,0 +1,15 @@ +from sqlmodel.cli import cli + + +def test_import_module() -> None: + assert len(cli.registered_groups) == 1 + assert cli.registered_groups[0].name == "migrations" + + +def test_import_module_with_fake_cli(fake_cli) -> None: + from sqlmodel.cli import cli, get_entry_points + + cli.registered_groups = [] + get_entry_points() + assert len(cli.registered_groups) == 2 + assert cli.registered_groups[0].name == "dummy" diff --git a/tests/test_cli/test_migrations.py b/tests/test_cli/test_migrations.py new file mode 100644 index 0000000000..440c1bcebb --- /dev/null +++ b/tests/test_cli/test_migrations.py @@ -0,0 +1,47 @@ +import configparser +from pathlib import Path + +from sqlmodel.cli import cli + + +def test_base_init(tmpdir, runner): + runner.invoke(cli, ["migrations", "init", "--module", str(tmpdir)]) + assert (Path(tmpdir) / "alembic.ini").exists() + + +def test_base_init_with_metadata(tmpdir, runner): + runner.invoke(cli, ["migrations", "init", "--module", str(tmpdir), "path.to.Model"]) + config = configparser.ConfigParser() + config.read(Path(tmpdir) / "alembic.ini") + + assert config["alembic"]["target_metadata"] == "path.to.Model" + + +def test_base_init_with_metadata_and_configfile(tmpdir, runner): + runner.invoke( + cli, + [ + "migrations", + "init", + "--module", + str(tmpdir), + "--config-file", + "foo.ini", + "path.to.Model", + ], + ) + config = configparser.ConfigParser() + config.read(Path(tmpdir) / "foo.ini") + + assert config["alembic"]["target_metadata"] == "path.to.Model" + + +def test_base_init_with_async_template(tmpdir, runner): + runner.invoke( + cli, + ["migrations", "init", "--module", str(tmpdir), "--template", "async"], + ) + with open(Path(tmpdir) / "migrations" / "README") as f: + assert ( + f.read() == "Generic single-database configuration with an async dbapi.\n" + )