From 313dc1525fb4fb2794a8da772916fae2bd6c37c2 Mon Sep 17 00:00:00 2001 From: ssantana-ns Date: Thu, 16 Sep 2021 10:19:08 -0600 Subject: [PATCH] Move to pynocular (#2) * Update code to ref pynocular and do readme * finished readme * Version bumped to 0.3.0 * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * Update README.md Co-authored-by: Jonathan Drake * PR comments * test issues * run pre-commit on readme * fix precommit hook * Update pynocular/__init__.py Co-authored-by: Jonathan Drake * update license * update year in license Co-authored-by: ns-circle-ci Co-authored-by: Jonathan Drake --- .flake8 | 2 +- LICENSE.md | 10 +- README.md | 185 +++++++++++++++++- ns_sql_utils/__init__.py | 5 - ns_sql_utils/config.py | 7 - pynocular/__init__.py | 5 + .../aiopg_transaction.py | 0 pynocular/config.py | 6 + {ns_sql_utils => pynocular}/database_model.py | 6 +- {ns_sql_utils => pynocular}/db_util.py | 4 +- {ns_sql_utils => pynocular}/engines.py | 4 +- {ns_sql_utils => pynocular}/exceptions.py | 0 pyproject.toml | 31 ++- setup.cfg | 2 +- tests/functional/test_database_model.py | 30 +-- 15 files changed, 232 insertions(+), 65 deletions(-) delete mode 100644 ns_sql_utils/__init__.py delete mode 100644 ns_sql_utils/config.py create mode 100644 pynocular/__init__.py rename {ns_sql_utils => pynocular}/aiopg_transaction.py (100%) create mode 100644 pynocular/config.py rename {ns_sql_utils => pynocular}/database_model.py (99%) rename {ns_sql_utils => pynocular}/db_util.py (97%) rename {ns_sql_utils => pynocular}/engines.py (99%) rename {ns_sql_utils => pynocular}/exceptions.py (100%) diff --git a/.flake8 b/.flake8 index 82842f2..3324322 100644 --- a/.flake8 +++ b/.flake8 @@ -8,7 +8,7 @@ ignore = E203,E501,E731,W503,W605 import-order-style = google # Packages added in this list should be added to the setup.cfg file as well application-import-names = - ns_sql_utils + pynocular exclude = *vendor* .venv diff --git a/LICENSE.md b/LICENSE.md index c250d86..c93d55d 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,5 +1,11 @@ Copyright (c) 2021, Narrative Science All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are NOT permitted. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + Neither the name of the nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/README.md b/README.md index 30dd56a..40f6a2e 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,21 @@ -# ns_sql_utils +# pynocular + +[![](https://img.shields.io/pypi/v/pynocular.svg)](https://pypi.org/pypi/pynocular/) [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) + +Pynocular is a lightweight ORM that lets you query your database using Pydantic models and asyncio. + +With Pynocular you can decorate your existing Pydantic models to sync them with the corresponding table in your +database, allowing you to persist changes without ever having to think about the database. Transaction management is +automatically handled for you so you can focus on the important parts of your code. This integrates seamlessly with frameworks that use Pydantic models such as FastAPI. -Utilities for interacting with SQL databases Features: -- +- Fully supports asyncio to write to SQL databases +- Provides simple methods for basic SQLAlchemy support (create, delete, update, read) +- Contains access to more advanced functionality such as custom SQLAlchemy selects +- Contains helper functions for creating new database tables +- Advanced transaction management system allows you to conditionally put requests in transactions Table of Contents: @@ -14,21 +25,179 @@ Table of Contents: ## Installation -ns_sql_utils requires Python 3.6 or above. +pynocular requires Python 3.6 or above. ```bash -pip install ns_sql_utils +pip install pynocular # or -poetry add ns_sql_utils +poetry add pynocular ``` ## Guide - +### Basic Usage +Pynocular works by decorating your base Pydantic model with the function `database_model`. Once decorated +with the proper information, you can proceed to use that model to interface with your specified database table. + +The first step is to define a `DBInfo` object. This will contain the connection information to your database. + +```python +from pynocular.engines import DatabaseType, DBInfo + + +# Example below shows how to connect to a locally-running Postgres database +connection_string = f"postgresql://{db_user_name}:{db_user_password}@localhost:5432/{db_name}?sslmode=disable" +) +db_info = DBInfo(DatabaseType.aiopg_engine, connection_string) +``` + +Pynocular supports connecting to your database through two different asyncio engines; aiopg and asyncpgsa. +You can pick which one you want to use by passing the correct `DatabaseType` enum value into `DBInfo`. + +Once you define a `db_info` object, you are ready to decorate your Pydantic models and interact with your database! + +```python +from pydantic import BaseModel, Field +from pynocular.database_model import database_model, UUID_STR + + +@database_model("organizations", db_info) +class Org(BaseModel): + + id: Optional[UUID_STR] = Field(primary_key=True, fetch_on_create=True) + serial_id: Optional[int] + name: str = Field(max_length=45) + slug: str = Field(max_length=45) + tag: Optional[str] = Field(max_length=100) + + created_at: Optional[datetime] = Field(fetch_on_create=True) + updated_at: Optional[datetime] = Field(fetch_on_update=True) + + +# Create a new Org via `create` +org = await Org.create("new org", "new-org") + + +# Create a new Org via `save` +org2 = Org("new org2", "new-org2") +await org2.save() + + +# Update an org +org.name = "renamed org" +await org.save() + + +# Delete org +await org.delete() + + +# Fetch org +org3 = await Org.get(org2.id) +assert org3 == org2 + +# Fetch a list of orgs +orgs = await Org.get_list() + +# Fetch a filtered list of orgs +orgs = await Org.get_list(tag="green") + +# Fetch orgs that have several different tags +orgs = await Org.get_list(tag=["green", "blue", "red"]) +``` + +With Pynocular you can set fields to be optional and set by the database. This is useful +if you want to let the database autogenerate your primary key or `created_at` and `updated_at` fields +on your table. To do this you must: +* Wrap the typehint in `Optional` +* Provide keyword arguments of `fetch_on_create=True` or `fetch_on_update=True` to the `Field` class + +### Advanced Usage +For most use cases, the basic usage defined above should suffice. However, there are certain situations +where you don't necessarily want to fetch each object or you need to do more complex queries that +are not exposed by the `DatabaseModel` interface. Below are some examples of how those situations can +be addressed using Pynocular. + +#### Batch operations on tables +Sometimes you want to insert a bunch of records into a database and you don't want to do an insert for each one. +This can be handled by the `create_list` function. + +```python +org_list = [ + Org(name="org1", slug="org-slug1"), + Org(name="org2", slug="org-slug2"), + Org(name="org3", slug="org-slug3"), +] +await Org.create_list(org_list) +``` +This function will insert all records into your database table in one batch. + + +If you have a use case that requires deleting a bunch of records based on some field value, you can use `delete_records`: + +```python +# Delete all records with the tag "green" +await Org.delete_records(tag="green") + +# Delete all records with if their tag has any of the following: "green", "blue", "red" +await Org.delete_records(tag=["green", "blue", "red"]) +``` + +Sometimes you may want to update the value of a record in a database without having to fetch it first. This can be accomplished by using +the `update_record` function: + +```python +await Org.update_record( + id="05c0060c-ceb8-40f0-8faa-dfb91266a6cf", + tag="blue" +) +org = await Org.get("05c0060c-ceb8-40f0-8faa-dfb91266a6cf") +assert org.tag == "blue" +``` + +#### Complex queries +Sometimes your application will require performing complex queries, such as getting the count of each unique field value for all records in the table. +Because Pynocular is backed by SQLAlchemy, we can access table columns directly to write pure SQLAlchemy queries as well! + +```python +from sqlalchemy import func, select +from pynocular.engines import DBEngine +async def generate_org_stats(): + query = ( + select([func.count(Org.column.id), Org.column.tag]) + .group_by(Org.column.tag) + .order_by(func.count().desc()) + ) + async with await DBEngine.transaction(Org._database_info, is_conditional=False) as conn: + result = await conn.execute(query) + return [dict(row) async for row in result] +``` +NOTE: `DBengine.transaction` is used to create a connection to the database using the credentials passed in. +If `is_conditional` is `False`, then it will add the query to any transaction that is opened in the call chain. This allows us to make database calls +in different functions but still have them all be under the same database transaction. If there is no transaction opened in the call chain it will open +a new one and any subsequent calls underneath that context manager will be added to the new transaction. + +If `is_conditional` is `True` and there is no transaction in the call chain, then the connection will not create a new transaction. Instead, the query will be performed without a transaction. + + +### Creating database tables +When you decorate a Pydantic model with Pynocular, it creates a SQLAlchemy table as a private variable. This can be accessed via the `_table` property +(although accessing private variables is not recommended). Using this, along with Pynocular's `create_tracked_table` function, allows you to create tables +in your database based off of Pydantic models! + +```python +from pynocular.db_utils import create_tracked_table + +from my_package import Org + +# Creates the table "organizations" in the database defined by db_info +await create_tracked_table(Org._database_info, Org._table) + +``` ## Development -To develop ns_sql_utils, install dependencies and enable the pre-commit hook: +To develop pynocular, install dependencies and enable the pre-commit hook: ```bash pip install pre-commit poetry diff --git a/ns_sql_utils/__init__.py b/ns_sql_utils/__init__.py deleted file mode 100644 index aa84b8a..0000000 --- a/ns_sql_utils/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Utilities for interacting with SQL databases""" - -__version__ = "0.2.0" - -from ns_sql_utils.engines import DatabaseType, DBInfo diff --git a/ns_sql_utils/config.py b/ns_sql_utils/config.py deleted file mode 100644 index 864b50a..0000000 --- a/ns_sql_utils/config.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Configuration for engines and models""" - -from ns_env_config import EnvConfig - -POOL_RECYCLE = EnvConfig.integer("POOL_RECYCLE", 300) -DB_POOL_MIN_SIZE = EnvConfig.integer("DB_POOL_MIN_SIZE", 2) -DB_POOL_MAX_SIZE = EnvConfig.integer("DB_POOL_MAX_SIZE", 10) diff --git a/pynocular/__init__.py b/pynocular/__init__.py new file mode 100644 index 0000000..61dd4d0 --- /dev/null +++ b/pynocular/__init__.py @@ -0,0 +1,5 @@ +"""Lightweight ORM that lets you query your database using Pydantic models and asyncio""" + +__version__ = "0.3.0" + +from pynocular.engines import DatabaseType, DBInfo diff --git a/ns_sql_utils/aiopg_transaction.py b/pynocular/aiopg_transaction.py similarity index 100% rename from ns_sql_utils/aiopg_transaction.py rename to pynocular/aiopg_transaction.py diff --git a/pynocular/config.py b/pynocular/config.py new file mode 100644 index 0000000..ab125c0 --- /dev/null +++ b/pynocular/config.py @@ -0,0 +1,6 @@ +"""Configuration for engines and models""" +import os + +POOL_RECYCLE = int(os.environ.get("POOL_RECYCLE", 300)) +DB_POOL_MIN_SIZE = int(os.environ.get("DB_POOL_MIN_SIZE", 2)) +DB_POOL_MAX_SIZE = int(os.environ.get("DB_POOL_MAX_SIZE", 10)) diff --git a/ns_sql_utils/database_model.py b/pynocular/database_model.py similarity index 99% rename from ns_sql_utils/database_model.py rename to pynocular/database_model.py index a4cd220..7cc56e1 100644 --- a/ns_sql_utils/database_model.py +++ b/pynocular/database_model.py @@ -25,8 +25,8 @@ from sqlalchemy.sql.base import ImmutableColumnCollection from sqlalchemy.sql.elements import BinaryExpression, UnaryExpression -from ns_sql_utils.engines import DBEngine, DBInfo -from ns_sql_utils.exceptions import ( +from pynocular.engines import DBEngine, DBInfo +from pynocular.exceptions import ( DatabaseModelMisconfigured, DatabaseModelMissingField, DatabaseRecordNotFound, @@ -53,8 +53,6 @@ def is_valid_uuid(string: str) -> bool: return False -# To be swapped once ns_data_structures exists -# from ns_data_structures.pydantic_types import UUID_STR class UUID_STR(str): """A string that represents a UUID4 value""" diff --git a/ns_sql_utils/db_util.py b/pynocular/db_util.py similarity index 97% rename from ns_sql_utils/db_util.py rename to pynocular/db_util.py index d8c5430..26027df 100644 --- a/ns_sql_utils/db_util.py +++ b/pynocular/db_util.py @@ -7,8 +7,8 @@ import sqlalchemy as sa from sqlalchemy.sql.ddl import CreateTable -from ns_sql_utils.engines import DatabaseType, DBEngine, DBInfo -from ns_sql_utils.exceptions import InvalidSqlIdentifierErr +from pynocular.engines import DatabaseType, DBEngine, DBInfo +from pynocular.exceptions import InvalidSqlIdentifierErr logger = logging.getLogger() diff --git a/ns_sql_utils/engines.py b/pynocular/engines.py similarity index 99% rename from ns_sql_utils/engines.py rename to pynocular/engines.py index 70e233c..980b52b 100644 --- a/ns_sql_utils/engines.py +++ b/pynocular/engines.py @@ -17,11 +17,11 @@ from sqlalchemy import create_engine as create_engine_sync from sqlalchemy.engine import Engine as SyncEngine -from ns_sql_utils.aiopg_transaction import ( +from pynocular.aiopg_transaction import ( ConditionalTransaction, transaction as Transaction, ) -from ns_sql_utils.config import DB_POOL_MAX_SIZE, DB_POOL_MIN_SIZE, POOL_RECYCLE +from pynocular.config import DB_POOL_MAX_SIZE, DB_POOL_MIN_SIZE, POOL_RECYCLE logger = logging.getLogger(__name__) diff --git a/ns_sql_utils/exceptions.py b/pynocular/exceptions.py similarity index 100% rename from ns_sql_utils/exceptions.py rename to pynocular/exceptions.py diff --git a/pyproject.toml b/pyproject.toml index 2632a7f..a6a41e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,36 +1,27 @@ [tool.poetry] -name = "ns_sql_utils" -version = "0.2.0" -description = "Utilities for interacting with SQL databases" -authors = ["Gregory Berns-Leone "] -license = "Proprietary" +name = "pynocular" +version = "0.3.0" +description = "Aysnchronous ORM framework for Pydantic Models" +authors = [ + "RJ Santana ", + "Patrick Hennessy ", + "Gregory Berns-Leone " +] +license = "BSD-3-Clause" readme = "README.md" -homepage = "https://github.com/NarrativeScience/ns_sql_utils" -repository = "https://github.com/NarrativeScience/ns_sql_utils" +homepage = "https://github.com/NarrativeScience/pynocular" +repository = "https://github.com/NarrativeScience/pynocular" [tool.poetry.dependencies] python = "^3.6.5" -# Up to "^3.1"? aenum = "^2.1" aiocontextvars = "=0.2.2" -# Can we unpin this? -# aiopg = ">=1.2" aiopg = "==0.16.0" -# Can we unpin this? -# asyncpg = "^0.23" asyncpg = "==0.15.0" asyncpgsa = "==0.23.0" -# Can we unpin this? -# backoff = "^1.11" backoff = "==1.10.0" -# Up to "^1.8"? pydantic = "^1.6" -# Can we unpin this? -# SQLAlchemy = "^1.4" SQLAlchemy = ">=1.3.19<1.4" -# Doesn't exist yet -# ns-data-structures = "" -ns-env-config = "^0.2.0" [tool.poetry.dev-dependencies] pre-commit = "^2.10.1" diff --git a/setup.cfg b/setup.cfg index 012e981..1cc771f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,7 @@ force_sort_within_sections=true include_trailing_comma=true known_standard_library=typing known_first_party= - ns_sql_utils + pynocular line_length=88 multi_line_output=3 no_lines_before=LOCALFOLDER diff --git a/tests/functional/test_database_model.py b/tests/functional/test_database_model.py index b2df70e..ec8cd76 100644 --- a/tests/functional/test_database_model.py +++ b/tests/functional/test_database_model.py @@ -2,30 +2,34 @@ import asyncio from asyncio import gather, sleep from datetime import datetime +import os from typing import Optional from uuid import uuid4 -from ns_env_config import EnvConfig from pydantic import BaseModel, Field from pydantic.error_wrappers import ValidationError import pytest -from ns_sql_utils.database_model import database_model, UUID_STR -from ns_sql_utils.db_util import add_trigger, create_new_database, create_table -from ns_sql_utils.engines import DatabaseType, DBEngine, DBInfo -from ns_sql_utils.exceptions import DatabaseModelMissingField, DatabaseRecordNotFound +from pynocular.database_model import database_model, UUID_STR +from pynocular.db_util import add_trigger, create_new_database, create_table +from pynocular.engines import DatabaseType, DBEngine, DBInfo +from pynocular.exceptions import DatabaseModelMissingField, DatabaseRecordNotFound -db_user_password = EnvConfig.string("DB_USER_PASSWORD") +db_user_password = str(os.environ.get("DB_USER_PASSWORD")) # DB to initially connect to so we can create a new db -existing_connection_string = EnvConfig.string( - "EXISTING_DB_CONNECTION_STRING", - f"postgresql://postgres:{db_user_password}@localhost:5432/postgres?sslmode=disable", +existing_connection_string = str( + os.environ.get( + "EXISTING_DB_CONNECTION_STRING", + f"postgresql://postgres:{db_user_password}@localhost:5432/postgres?sslmode=disable", + ) ) -test_db_name = EnvConfig.string("TEST_DB_NAME", "test_db") -test_connection_string = EnvConfig.string( - "TEST_DB_CONNECTION_STRING", - f"postgresql://postgres:{db_user_password}@localhost:5432/{test_db_name}?sslmode=disable", +test_db_name = str(os.environ.get("TEST_DB_NAME", "test_db")) +test_connection_string = str( + os.environ.get( + "TEST_DB_CONNECTION_STRING", + f"postgresql://postgres:{db_user_password}@localhost:5432/{test_db_name}?sslmode=disable", + ) ) testdb = DBInfo(DatabaseType.aiopg_engine, test_connection_string)