Merge pull request #17 from great-expectations/m/CORE-730/optional-deps

[MAINTENANCE] Add extras to setup.cfg for optional deps

Author: tyler-hoffman

.github/workflows/integration-test.yml

@@ -41,10 +41,7 @@ jobs:
         run: python -m pip install --upgrade pip
 
       - name: Install Library
-        run: pip install .
-
-      - name: Install Dependencies
-        run: pip install -r test-requirements.txt
+        run: pip install .[postgresql,tests]
 
       - name: Setup
         run: |
@@ -78,10 +75,7 @@ jobs:
         run: python -m pip install --upgrade pip
 
       - name: Install Library
-        run: pip install .
-
-      - name: Install Dependencies
-        run: pip install -r test-requirements.txt
+        run: pip install .[tests,spark]
 
       - name: Setup
         run: |
@@ -115,10 +109,7 @@ jobs:
         run: python -m pip install --upgrade pip
 
       - name: Install Library
-        run: pip install .
-
-      - name: Install Dependencies
-        run: pip install -r test-requirements.txt
+        run: pip install .[tests,spark]
 
       - name: Setup
         run: |

.github/workflows/lint.yml

@@ -20,10 +20,7 @@ jobs:
           python-version: 3.12
 
       - name: Install Library
-        run: pip install .
-
-      - name: Install Dependencies
-        run: pip install -r test-requirements.txt
+        run: pip install .[lint]
 
       - name: Ruff Formatter
         run: ruff format --check .

.github/workflows/unit-test.yml

@@ -27,10 +27,7 @@ jobs:
         run: python -m pip install --upgrade pip
 
       - name: Install Library
-        run: pip install .
-
-      - name: Install Dependencies
-        run: pip install -r test-requirements.txt
+        run: pip install .[tests]
 
       - name: Run Unit Tests
         run: pytest -vvv -m unit tests/unit

setup.cfg

@@ -6,14 +6,41 @@ python_requires = >=3.9
 packages = find_namespace:
 include_package_data = true
 install_requires =
+    great-expectations>=1.3.1
     apache-airflow>=2.1
-    great-expectations[snowflake,postgresql,mssql,bigquery,athena,spark,gcp,azure,s3]>=1.3.1
     setuptools>=41.0.0
 
 [options.extras_require]
+athena =
+    great-expectations[athena]>=1.3.1
+azure =
+    great-expectations[azure]>=1.3.1
+bigquery =
+    great-expectations[bigquery]>=1.3.1
+lint =
+    mypy==1.14.1
+    ruff==0.8.3
+    pytest==8.3.4
+    pytest-mock==3.14.0
+    great-expectations[spark, spark-connect]>=1.3.1
+gcp =
+    great-expectations[gcp]>=1.3.1
+mssql =
+    great-expectations[mssql]>=1.3.1
+postgresql =
+    great-expectations[postgresql]>=1.3.1
+s3 =
+    great-expectations[s3]>=1.3.1
+snowflake =
+    great-expectations[snowflake]>=1.3.1
+spark =
+    great-expectations[spark, spark-connect]>=1.3.1
+    pyarrow>=4.0.0
 tests =
-    pytest
+    pytest==8.3.4
+    pytest-mock==3.14.0
 
 [options.entry_points]
 apache_airflow_provider=
   provider_info=great_expectations_provider.__init__:get_provider_info
+

test-requirements.txt

@@ -1,18 +1,21 @@
-from typing import Callable
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Callable
 
 import pandas as pd
-import pyspark.sql as pyspark
 import pytest
 from great_expectations import ExpectationSuite
 from great_expectations.expectations import ExpectColumnValuesToBeInSet
-from pyspark.sql.connect.dataframe import DataFrame as SparkConnectDataFrame
-from pyspark.sql.connect.session import SparkSession as SparkConnectSession
 
 from great_expectations_provider.operators.validate_dataframe import (
     GXValidateDataFrameOperator,
 )
 from integration.conftest import is_valid_gx_cloud_url, rand_name
 
+if TYPE_CHECKING:
+    from pyspark.sql import SparkSession
+    from pyspark.sql.connect.session import SparkSession as SparkConnectSession
+
 
 class TestGXValidateDataFrameOperator:
     @pytest.mark.integration
@@ -57,7 +60,9 @@ def configure_dataframe() -> pd.DataFrame:
         assert is_valid_gx_cloud_url(result["result_url"])
 
     @pytest.mark.spark_integration
-    def test_spark(self, spark_session: pyspark.SparkSession) -> None:
+    def test_spark(self, spark_session: SparkSession) -> None:
+        import pyspark.sql as pyspark
+
         column_name = "col_A"
         task_id = f"test_spark_{rand_name()}"
 
@@ -85,6 +90,8 @@ def configure_dataframe() -> pyspark.DataFrame:
 
     @pytest.mark.spark_connect_integration
     def test_spark_connect(self, spark_connect_session: SparkConnectSession) -> None:
+        from pyspark.sql.connect.dataframe import DataFrame as SparkConnectDataFrame
+
         column_name = "col_A"
         task_id = f"test_spark_{rand_name()}"
 
@@ -112,14 +119,19 @@ def configure_dataframe() -> SparkConnectDataFrame:
 
 
 @pytest.fixture
-def spark_session() -> pyspark.SparkSession:
+def spark_session() -> SparkSession:
+    import pyspark.sql as pyspark
+
     session = pyspark.SparkSession.builder.getOrCreate()
     assert isinstance(session, pyspark.SparkSession)
     return session
 
 
 @pytest.fixture
 def spark_connect_session() -> SparkConnectSession:
+    import pyspark.sql as pyspark
+    from pyspark.sql.connect.session import SparkSession as SparkConnectSession
+
     session = pyspark.SparkSession.builder.remote("sc://localhost:15002").getOrCreate()
     assert isinstance(session, SparkConnectSession)
     return session

tests/integration/test_validate_dataframe_operator.py

@@ -13,7 +13,6 @@
 from great_expectations.expectations import (
     ExpectColumnValuesToBeInSet,
 )
-from pytest_mock import MockerFixture
 
 from great_expectations_provider.operators.validate_batch import GXValidateBatchOperator