Move to pynocular (#2)

* Update code to ref pynocular and do readme * finished readme * Version bumped to 0.3.0 * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * Update README.md Co-authored-by: Jonathan Drake <[email protected]> * PR comments * test issues * run pre-commit on readme * fix precommit hook * Update pynocular/__init__.py Co-authored-by: Jonathan Drake <[email protected]> * update license * update year in license Co-authored-by: ns-circle-ci <[email protected]> Co-authored-by: Jonathan Drake <[email protected]>
NarrativeScience-old · Sep 16, 2021 · 313dc15 · 313dc15
1 parent 6309297
commit 313dc15
Show file tree

Hide file tree

Showing 15 changed files with 232 additions and 65 deletions.
diff --git a/.flake8 b/.flake8
@@ -8,7 +8,7 @@ ignore = E203,E501,E731,W503,W605
 import-order-style = google
 # Packages added in this list should be added to the setup.cfg file as well
 application-import-names =
-    ns_sql_utils
+    pynocular
 exclude =
     *vendor*
     .venv

diff --git a/LICENSE.md b/LICENSE.md
@@ -1,5 +1,11 @@
 Copyright (c) 2021, Narrative Science
 
 All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are NOT permitted.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+    Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+    Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+    Neither the name of the nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/README.md b/README.md
@@ -1,10 +1,21 @@
-# ns_sql_utils
+# pynocular
+
+[![](https://img.shields.io/pypi/v/pynocular.svg)](https://pypi.org/pypi/pynocular/) [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause)
+
+Pynocular is a lightweight ORM that lets you query your database using Pydantic models and asyncio.
+
+With Pynocular you can decorate your existing Pydantic models to sync them with the corresponding table in your
+database, allowing you to persist changes without ever having to think about the database. Transaction management is
+automatically handled for you so you can focus on the important parts of your code. This integrates seamlessly with frameworks that use Pydantic models such as FastAPI.
 
-Utilities for interacting with SQL databases
 
 Features:
 
-- <!-- list of features -->
+- Fully supports asyncio to write to SQL databases
+- Provides simple methods for basic SQLAlchemy support (create, delete, update, read)
+- Contains access to more advanced functionality such as custom SQLAlchemy selects
+- Contains helper functions for creating new database tables
+- Advanced transaction management system allows you to conditionally put requests in transactions
 
 Table of Contents:
 
@@ -14,21 +25,179 @@ Table of Contents:
 
 ## Installation
 
-ns_sql_utils requires Python 3.6 or above.
+pynocular requires Python 3.6 or above.
 
 ```bash
-pip install ns_sql_utils
+pip install pynocular
 # or
-poetry add ns_sql_utils
+poetry add pynocular
 ```
 
 ## Guide
 
-<!-- Subsections explaining how to use the package -->
+### Basic Usage
+Pynocular works by decorating your base Pydantic model with the function `database_model`. Once decorated
+with the proper information, you can proceed to use that model to interface with your specified database table.
+
+The first step is to define a `DBInfo` object. This will contain the connection information to your database.
+
+```python
+from pynocular.engines import DatabaseType, DBInfo
+
+
+# Example below shows how to connect to a locally-running Postgres database
+connection_string = f"postgresql://{db_user_name}:{db_user_password}@localhost:5432/{db_name}?sslmode=disable"
+)
+db_info = DBInfo(DatabaseType.aiopg_engine, connection_string)
+```
+
+Pynocular supports connecting to your database through two different asyncio engines; aiopg and asyncpgsa.
+You can pick which one you want to use by passing the correct `DatabaseType` enum value into `DBInfo`.
+
+Once you define a `db_info` object, you are ready to decorate your Pydantic models and interact with your database!
+
+```python
+from pydantic import BaseModel, Field
+from pynocular.database_model import database_model, UUID_STR
+
+
+@database_model("organizations", db_info)
+class Org(BaseModel):
+
+    id: Optional[UUID_STR] = Field(primary_key=True, fetch_on_create=True)
+    serial_id: Optional[int]
+    name: str = Field(max_length=45)
+    slug: str = Field(max_length=45)
+    tag: Optional[str] = Field(max_length=100)
+
+    created_at: Optional[datetime] = Field(fetch_on_create=True)
+    updated_at: Optional[datetime] = Field(fetch_on_update=True)
+
+
+# Create a new Org via `create`
+org = await Org.create("new org", "new-org")
+
+
+# Create a new Org via `save`
+org2 = Org("new org2", "new-org2")
+await org2.save()
+
+
+# Update an org
+org.name = "renamed org"
+await org.save()
+
+
+# Delete org
+await org.delete()
+
+
+# Fetch org
+org3 = await Org.get(org2.id)
+assert org3 == org2
+
+# Fetch a list of orgs
+orgs = await Org.get_list()
+
+# Fetch a filtered list of orgs
+orgs = await Org.get_list(tag="green")
+
+#  Fetch orgs that have several different tags
+orgs = await Org.get_list(tag=["green", "blue", "red"])
+```
+
+With Pynocular you can set fields to be optional and set by the database. This is useful
+if you want to let the database autogenerate your primary key or `created_at` and `updated_at` fields
+on your table. To do this you must:
+* Wrap the typehint in `Optional`
+* Provide keyword arguments of `fetch_on_create=True` or `fetch_on_update=True` to the `Field` class
+
+### Advanced Usage
+For most use cases, the basic usage defined above should suffice. However, there are certain situations
+where you don't necessarily want to fetch each object or you need to do more complex queries that
+are not exposed by the `DatabaseModel` interface. Below are some examples of how those situations can
+be addressed using Pynocular.
+
+#### Batch operations on tables
+Sometimes you want to insert a bunch of records into a database and you don't want to do an insert for each one.
+This can be handled by the `create_list` function.
+
+```python
+org_list = [
+    Org(name="org1", slug="org-slug1"),
+    Org(name="org2", slug="org-slug2"),
+    Org(name="org3", slug="org-slug3"),
+]
+await Org.create_list(org_list)
+```
+This function will insert all records into your database table in one batch.
+
+
+If you have a use case that requires deleting a bunch of records based on some field value, you can use `delete_records`:
+
+```python
+# Delete all records with the tag "green"
+await Org.delete_records(tag="green")
+
+# Delete all records with if their tag has any of the following: "green", "blue", "red"
+await Org.delete_records(tag=["green", "blue", "red"])
+```
+
+Sometimes you may want to update the value of a record in a database without having to fetch it first. This can be accomplished by using
+the `update_record` function:
+
+```python
+await Org.update_record(
+    id="05c0060c-ceb8-40f0-8faa-dfb91266a6cf",
+    tag="blue"
+)
+org = await Org.get("05c0060c-ceb8-40f0-8faa-dfb91266a6cf")
+assert org.tag == "blue"
+```
+
+#### Complex queries
+Sometimes your application will require performing complex queries, such as getting the count of each unique field value for all records in the table.
+Because Pynocular is backed by SQLAlchemy, we can access table columns directly to write pure SQLAlchemy queries as well!
+
+```python
+from sqlalchemy import func, select
+from pynocular.engines import DBEngine
+async def generate_org_stats():
+    query = (
+        select([func.count(Org.column.id), Org.column.tag])
+        .group_by(Org.column.tag)
+        .order_by(func.count().desc())
+    )
+    async with await DBEngine.transaction(Org._database_info, is_conditional=False) as conn:
+        result = await conn.execute(query)
+        return [dict(row) async for row in result]
+```
+NOTE: `DBengine.transaction` is used to create a connection to the database using the credentials passed in.
+If `is_conditional` is `False`, then it will add the query to any transaction that is opened in the call chain. This allows us to make database calls
+in different functions but still have them all be under the same database transaction. If there is no transaction opened in the call chain it will open
+a new one and any subsequent calls underneath that context manager will be added to the new transaction.
+
+If `is_conditional` is `True` and there is no transaction in the call chain, then the connection will not create a new transaction. Instead, the query will be performed without a transaction.
+
+
+### Creating database tables
+When you decorate a Pydantic model with Pynocular, it creates a SQLAlchemy table as a private variable. This can be accessed via the `_table` property
+(although accessing private variables is not recommended). Using this, along with Pynocular's `create_tracked_table` function, allows you to create tables
+in your database based off of Pydantic models!
+
+```python
+from pynocular.db_utils import create_tracked_table
+
+from my_package import Org
+
+# Creates the table "organizations" in the database defined by db_info
+await create_tracked_table(Org._database_info, Org._table)
+
+```
 
 ## Development
 
-To develop ns_sql_utils, install dependencies and enable the pre-commit hook:
+To develop pynocular, install dependencies and enable the pre-commit hook:
 
 ```bash
 pip install pre-commit poetry

diff --git a/ns_sql_utils/__init__.py b/ns_sql_utils/__init__.py
diff --git a/ns_sql_utils/config.py b/ns_sql_utils/config.py
diff --git a/pynocular/__init__.py b/pynocular/__init__.py
@@ -0,0 +1,5 @@
+"""Lightweight ORM that lets you query your database using Pydantic models and asyncio"""
+
+__version__ = "0.3.0"
+
+from pynocular.engines import DatabaseType, DBInfo
diff --git a/ns_sql_utils/aiopg_transaction.py → pynocular/aiopg_transaction.py b/ns_sql_utils/aiopg_transaction.py → pynocular/aiopg_transaction.py
diff --git a/pynocular/config.py b/pynocular/config.py
@@ -0,0 +1,6 @@
+"""Configuration for engines and models"""
+import os
+
+POOL_RECYCLE = int(os.environ.get("POOL_RECYCLE", 300))
+DB_POOL_MIN_SIZE = int(os.environ.get("DB_POOL_MIN_SIZE", 2))
+DB_POOL_MAX_SIZE = int(os.environ.get("DB_POOL_MAX_SIZE", 10))
diff --git a/ns_sql_utils/database_model.py → pynocular/database_model.py b/ns_sql_utils/database_model.py → pynocular/database_model.py
@@ -25,8 +25,8 @@
 from sqlalchemy.sql.base import ImmutableColumnCollection
 from sqlalchemy.sql.elements import BinaryExpression, UnaryExpression
 
-from ns_sql_utils.engines import DBEngine, DBInfo
-from ns_sql_utils.exceptions import (
+from pynocular.engines import DBEngine, DBInfo
+from pynocular.exceptions import (
     DatabaseModelMisconfigured,
     DatabaseModelMissingField,
     DatabaseRecordNotFound,
@@ -53,8 +53,6 @@ def is_valid_uuid(string: str) -> bool:
         return False
 
 
-# To be swapped once ns_data_structures exists
-# from ns_data_structures.pydantic_types import UUID_STR
 class UUID_STR(str):
     """A string that represents a UUID4 value"""
 

diff --git a/ns_sql_utils/db_util.py → pynocular/db_util.py b/ns_sql_utils/db_util.py → pynocular/db_util.py
@@ -7,8 +7,8 @@
 import sqlalchemy as sa
 from sqlalchemy.sql.ddl import CreateTable
 
-from ns_sql_utils.engines import DatabaseType, DBEngine, DBInfo
-from ns_sql_utils.exceptions import InvalidSqlIdentifierErr
+from pynocular.engines import DatabaseType, DBEngine, DBInfo
+from pynocular.exceptions import InvalidSqlIdentifierErr
 
 logger = logging.getLogger()
 

diff --git a/ns_sql_utils/engines.py → pynocular/engines.py b/ns_sql_utils/engines.py → pynocular/engines.py
@@ -17,11 +17,11 @@
 from sqlalchemy import create_engine as create_engine_sync
 from sqlalchemy.engine import Engine as SyncEngine
 
-from ns_sql_utils.aiopg_transaction import (
+from pynocular.aiopg_transaction import (
     ConditionalTransaction,
     transaction as Transaction,
 )
-from ns_sql_utils.config import DB_POOL_MAX_SIZE, DB_POOL_MIN_SIZE, POOL_RECYCLE
+from pynocular.config import DB_POOL_MAX_SIZE, DB_POOL_MIN_SIZE, POOL_RECYCLE
 
 logger = logging.getLogger(__name__)
 

diff --git a/ns_sql_utils/exceptions.py → pynocular/exceptions.py b/ns_sql_utils/exceptions.py → pynocular/exceptions.py
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,36 +1,27 @@
 [tool.poetry]
-name = "ns_sql_utils"
-version = "0.2.0"
-description = "Utilities for interacting with SQL databases"
-authors = ["Gregory Berns-Leone <[email protected]>"]
-license = "Proprietary"
+name = "pynocular"
+version = "0.3.0"
+description = "Aysnchronous ORM framework for Pydantic Models"
+authors = [
+  "RJ Santana <[email protected]>",
+  "Patrick Hennessy <[email protected]>",
+  "Gregory Berns-Leone <[email protected]>"
+]
+license = "BSD-3-Clause"
 readme = "README.md"
-homepage = "https://github.com/NarrativeScience/ns_sql_utils"
-repository = "https://github.com/NarrativeScience/ns_sql_utils"
+homepage = "https://github.com/NarrativeScience/pynocular"
+repository = "https://github.com/NarrativeScience/pynocular"
 
 [tool.poetry.dependencies]
 python = "^3.6.5"
-# Up to "^3.1"?
 aenum = "^2.1"
 aiocontextvars = "=0.2.2"
-# Can we unpin this?
-# aiopg = ">=1.2"
 aiopg = "==0.16.0"
-# Can we unpin this?
-# asyncpg = "^0.23"
 asyncpg = "==0.15.0"
 asyncpgsa = "==0.23.0"
-# Can we unpin this?
-# backoff = "^1.11"
 backoff = "==1.10.0"
-# Up to "^1.8"?
 pydantic = "^1.6"
-# Can we unpin this?
-# SQLAlchemy = "^1.4"
 SQLAlchemy = ">=1.3.19<1.4"
-# Doesn't exist yet
-# ns-data-structures = ""
-ns-env-config = "^0.2.0"
 
 [tool.poetry.dev-dependencies]
 pre-commit = "^2.10.1"

diff --git a/setup.cfg b/setup.cfg
@@ -10,7 +10,7 @@ force_sort_within_sections=true
 include_trailing_comma=true
 known_standard_library=typing
 known_first_party=
-    ns_sql_utils
+    pynocular
 line_length=88
 multi_line_output=3
 no_lines_before=LOCALFOLDER