initial cut of asset model

src/fides/api/alembic/migrations/versions/021166731846_add_asset_table.py

@@ -0,0 +1,101 @@
+"""add asset table
+
+Revision ID: 021166731846
+Revises: 58f8edd66b69
+Create Date: 2025-01-22 22:14:35.548869
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "021166731846"
+down_revision = "58f8edd66b69"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # stored separately to be referenced in md5 expression on index
+
+    asset_table = op.create_table(
+        "asset",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("asset_type", sa.String(), nullable=False),
+        sa.Column("domain", sa.String(), nullable=True),
+        sa.Column("parent", sa.String(), nullable=True),
+        sa.Column("parent_domain", sa.String(), nullable=True),
+        sa.Column(
+            "locations",
+            postgresql.ARRAY(sa.String()),
+            server_default="{}",
+            nullable=False,
+        ),
+        sa.Column("with_consent", sa.BOOLEAN(), nullable=False),
+        sa.Column(
+            "data_uses",
+            postgresql.ARRAY(sa.String()),
+            server_default="{}",
+            nullable=False,
+        ),
+        sa.Column(
+            "meta",
+            postgresql.JSONB(astext_type=sa.Text()),
+            server_default="{}",
+            nullable=False,
+        ),
+        sa.Column("path", sa.String(), nullable=True),
+        sa.Column("base_url", sa.String(), nullable=True),
+        sa.Column("system_id", sa.String(), nullable=True),
+        sa.ForeignKeyConstraint(["system_id"], ["ctl_systems.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_asset_asset_type"), "asset", ["asset_type"], unique=False)
+    op.create_index(op.f("ix_asset_domain"), "asset", ["domain"], unique=False)
+    op.create_index(op.f("ix_asset_id"), "asset", ["id"], unique=False)
+    op.create_index(op.f("ix_asset_name"), "asset", ["name"], unique=False)
+    op.create_index(op.f("ix_asset_system_id"), "asset", ["system_id"], unique=False)
+
+    op.create_index(
+        op.f("ix_asset_name_asset_type_domain_base_url_system_id"),
+        "asset",
+        [
+            "name",
+            "asset_type",
+            "domain",
+            sa.text("coalesce(md5(base_url), 'NULL')"),
+            "system_id",
+        ],
+        unique=True,
+    )
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_asset_system_id"), table_name="asset")
+    op.drop_index(op.f("ix_asset_name"), table_name="asset")
+    op.drop_index(op.f("ix_asset_id"), table_name="asset")
+    op.drop_index(op.f("ix_asset_domain"), table_name="asset")
+    op.drop_index(op.f("ix_asset_asset_type"), table_name="asset")
+    op.drop_index(
+        op.f("ix_asset_name_asset_type_domain_base_url_system_id"), table_name="asset"
+    )
+    op.drop_table("asset")
+    # ### end Alembic commands ###

src/fides/api/db/base.py

@@ -3,6 +3,7 @@
 # imported by Alembic
 from fides.api.db.base_class import Base
 from fides.api.models.application_config import ApplicationConfig
+from fides.api.models.asset import Asset
 from fides.api.models.audit_log import AuditLog
 from fides.api.models.authentication_request import AuthenticationRequest
 from fides.api.models.client import ClientDetail

src/fides/api/models/asset.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Type
+
+from sqlalchemy import (
+    ARRAY,
+    BOOLEAN,
+    Column,
+    ForeignKey,
+    Index,
+    String,
+    func,
+    insert,
+    select,
+    update,
+)
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.ext.mutable import MutableDict
+from sqlalchemy.orm import relationship
+
+from fides.api.db.base_class import Base
+from fides.api.models.sql_models import System
+
+
+class Asset(Base):
+    """
+    Web assets associated with a system. This model will supersede `Cookies` once we have established
+    a migration path and backward compatibility with all `Cookies` related APIs.
+    """
+
+    # Common attributes
+    name = Column(String, index=True, nullable=False)
+    asset_type = Column(String, index=True, nullable=False)
+    domain = Column(String, index=True)
+    parent = Column(String)
+    parent_domain = Column(String)
+    locations = Column(ARRAY(String), server_default="{}", nullable=False)
+    with_consent = Column(BOOLEAN, default=False, nullable=False)
+    data_uses = Column(ARRAY(String), server_default="{}", nullable=False)
+
+    # generic object to store additional attributes, specific to asset type
+    meta = Column(
+        MutableDict.as_mutable(JSONB),
+        nullable=False,
+        server_default="{}",
+        default=dict,
+    )
+
+    # Cookie-specific attributes
+    path = Column(String)
+
+    # Browser request-specific attributes
+    base_url = Column(String)
+
+    system_id = Column(
+        String, ForeignKey(System.id_field_path, ondelete="CASCADE"), index=True
+    )  # If system is deleted, remove the associated assets.
+
+    system = relationship(
+        System,
+        back_populates="assets",
+        cascade="all,delete",
+        uselist=False,
+        lazy="selectin",
+    )
+
+    # we need to use an md5 of the base_url to avoid constraint/index length issues
+    # and we need to use a unique index, rather than constraint, since postgresql constraints
+    # do not support expressions, only direct column references
+    __table_args__ = (
+        Index(
+            "ix_asset_name_asset_type_domain_base_url_system_id",
+            name,
+            asset_type,
+            domain,
+            func.coalesce(func.md5(base_url), "NULL"),
+            system_id,
+            unique=True,
+        ),
+    )
+
+    @classmethod
+    async def upsert_async(
+        cls: Type[Asset],
+        async_session: AsyncSession,
+        *,
+        data: Dict[str, Any],
+    ) -> Asset:
+        """
+        Creates a new Asset record if it does not exist, otherwise updates the existing Asset record
+        with the attribute values provided in the `data` dict.
+
+        Assets are looked up by the provided attributes that make up their uniqueness criteria:
+        - name
+        - asset_type
+        - domain
+        - base_url (if applicable)
+        - system_id.
+        """
+        if (
+            "name" not in data
+            or "asset_type" not in data
+            or "domain" not in data
+            or "system_id" not in data
+        ):
+            raise ValueError(
+                "name, asset_type, domain, and system_id are required fields on assets"
+            )
+
+        result = await async_session.execute(
+            select(cls).where(
+                cls.name == data["name"],
+                cls.asset_type == data["asset_type"],
+                cls.domain == data["domain"],
+                cls.system_id == data["system_id"],
+                cls.base_url == data.get("base_url"),
+            )
+        )  # type: ignore[arg-type]
+        existing_record = result.scalars().first()
+        record_id: str
+        if existing_record:
+            await async_session.execute(
+                update(cls).where(cls.id == existing_record.id).values(data)  # type: ignore[arg-type]
+            )
+            record_id = existing_record.id
+        else:
+            result = await async_session.execute(insert(cls).values(data))  # type: ignore[arg-type]
+            record_id = result.inserted_primary_key.id
+
+        result = await async_session.execute(select(cls).where(cls.id == record_id))  # type: ignore[arg-type]
+        return result.scalars().first()
+
+    @classmethod
+    async def get_by_system_async(
+        cls: Type[Asset],
+        async_session: AsyncSession,
+        system_id: Optional[str] = None,
+        system_fides_key: Optional[str] = None,
+    ) -> list[Asset]:
+        """
+        Retrieves all assets associated with a given system,
+        using the provided system `id` or `fides_key`, whichever is provided
+        """
+        if system_id:
+            query = select(cls).where(cls.system_id == system_id)
+        else:
+            if not system_fides_key:
+                raise ValueError(
+                    "Either system_id or system_fides_key must be provided"
+                )
+            query = (
+                select(cls)
+                .join(System, System.id == cls.system_id)
+                .where(System.fides_key == system_fides_key)
+            )
+
+        result = await async_session.execute(query)
+        return result.scalars().all()

src/fides/api/models/sql_models.py

@@ -437,6 +437,10 @@ class System(Base, FidesBase):
         "Cookies", back_populates="system", lazy="selectin", uselist=True, viewonly=True
     )
 
+    assets = relationship(
+        "Asset", back_populates="system", lazy="selectin", uselist=True, viewonly=True
+    )
+
     @classmethod
     def get_data_uses(
         cls: Type[System], systems: List[System], include_parents: bool = True

tests/conftest.py

@@ -13,6 +13,7 @@
 from fastapi import Query
 from fastapi.testclient import TestClient
 from fideslang import DEFAULT_TAXONOMY, models
+from fideslang.models import System as SystemSchema
 from httpx import AsyncClient
 from loguru import logger
 from sqlalchemy.engine.base import Engine
@@ -28,6 +29,7 @@
     JWE_PAYLOAD_SYSTEMS,
 )
 from fides.api.db.ctl_session import sync_engine
+from fides.api.db.system import create_system
 from fides.api.main import app
 from fides.api.models.privacy_request import (
     EXITED_EXECUTION_LOG_STATUSES,
@@ -1271,6 +1273,24 @@ def system(db: Session) -> System:
     return system
 
 
+@pytest.fixture()
+@pytest.mark.asyncio
+async def system_async(async_session):
+    """Creates a system for testing with an async session, to be used in async tests"""
+    resource = SystemSchema(
+        fides_key=str(uuid4()),
+        organization_fides_key="default_organization",
+        name="test_system_1",
+        system_type="test",
+        privacy_declarations=[],
+    )
+
+    system = await create_system(
+        resource, async_session, CONFIG.security.oauth_root_client_id
+    )
+    return system
+
+
 @pytest.fixture(scope="function")
 def system_hidden(db: Session) -> Generator[System, None, None]:
     system = System.create(