no browser for CI testing and fixing unit tests

Author: acrylJonny

metadata-ingestion/tests/integration/airbyte/test_airbyte.py

@@ -112,6 +112,7 @@ def airbyte_service(
         "--port",
         str(AIRBYTE_API_PORT),
         "--low-resource-mode",
+        "--no-browser",
     ]
 
     try:

metadata-ingestion/tests/unit/airbyte/test_airbyte_client.py

@@ -104,14 +104,14 @@ def test_init_with_api_key(self):
         assert client.config.api_key.get_secret_value() == "test-api-key"
 
     def test_init_without_host_port(self):
-        # Missing host_port should raise ValueError
-        config = AirbyteClientConfig(
-            deployment_type=AirbyteDeploymentType.OPEN_SOURCE,
-            # Missing host_port
-        )
+        # Missing host_port should raise ValidationError during config creation
+        from pydantic import ValidationError
 
-        with pytest.raises(ValueError, match="host_port is required"):
-            AirbyteOSSClient(config)
+        with pytest.raises(ValidationError, match="host_port is required"):
+            AirbyteClientConfig(
+                deployment_type=AirbyteDeploymentType.OPEN_SOURCE,
+                # Missing host_port
+            )
 
     @patch("datahub.ingestion.source.airbyte.client.AirbyteOSSClient._paginate_results")
     def test_list_workspaces(self, mock_paginate_results):
@@ -499,17 +499,17 @@ def test_init_with_defaults(self):
             assert client.config.oauth2_refresh_token is not None
 
     def test_init_missing_workspace_id(self):
-        # Missing workspace_id should raise ValueError
-        config = AirbyteClientConfig(
-            deployment_type=AirbyteDeploymentType.CLOUD,
-            oauth2_client_id="client-id",
-            oauth2_client_secret=SecretStr("client-secret"),
-            oauth2_refresh_token=SecretStr("refresh-token"),
-            # Missing cloud_workspace_id
-        )
+        # Missing workspace_id should raise ValidationError during config creation
+        from pydantic import ValidationError
 
-        with pytest.raises(ValueError, match="Workspace ID is required"):
-            AirbyteCloudClient(config)
+        with pytest.raises(ValidationError, match="cloud_workspace_id is required"):
+            AirbyteClientConfig(
+                deployment_type=AirbyteDeploymentType.CLOUD,
+                oauth2_client_id="client-id",
+                oauth2_client_secret=SecretStr("client-secret"),
+                oauth2_refresh_token=SecretStr("refresh-token"),
+                # Missing cloud_workspace_id
+            )
 
     @patch("requests.post")
     def test_refresh_oauth_token(self, mock_post):

metadata-ingestion/tests/unit/airbyte/test_models.py

@@ -1,20 +1,20 @@
 import unittest
-from enum import Enum
 from typing import Dict, List, Optional, Union
 
 import pytest
-from pydantic import BaseModel, Field, ValidationError
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
 
 from datahub.ingestion.source.airbyte.models import (
     AirbyteConnection,
     AirbyteDataSource,
     AirbyteDestination,
     AirbyteWorkspace,
 )
+from datahub.utilities.str_enum import StrEnum
 
 
 # Mock classes for testing purposes that don't exist in the models.py
-class AirbyteFieldType(str, Enum):
+class AirbyteFieldType(StrEnum):
     """Mock enum for field types in tests."""
 
     STRING = "string"
@@ -79,7 +79,7 @@ def to_schema_fields(
         return fields
 
 
-class AirbyteSyncMode(str, Enum):
+class AirbyteSyncMode(StrEnum):
     """Mock enum for sync modes in tests."""
 
     FULL_REFRESH = "full_refresh"
@@ -88,15 +88,15 @@ class AirbyteSyncMode(str, Enum):
     OVERWRITE = "overwrite"
 
 
-class AirbyteStatus(str, Enum):
+class AirbyteStatus(StrEnum):
     """Mock enum for connection status in tests."""
 
     ACTIVE = "active"
     INACTIVE = "inactive"
     DEPRECATED = "deprecated"
 
 
-class AirbyteConnectionScheduleType(str, Enum):
+class AirbyteConnectionScheduleType(StrEnum):
     """Mock enum for connection schedule types in tests."""
 
     MANUAL = "manual"
@@ -107,27 +107,21 @@ class AirbyteConnectionScheduleType(str, Enum):
 class AirbyteConnectionStream(BaseModel):
     """Mock class for connection stream in tests."""
 
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+
     name: str
     namespace: Optional[str] = None
     sync_mode: AirbyteSyncMode = AirbyteSyncMode.FULL_REFRESH
-    cursor_field: List[str] = Field(default_factory=list)
-    primary_key: List[List[str]] = Field(default_factory=list)
-    destination_sync_mode: AirbyteSyncMode = AirbyteSyncMode.OVERWRITE
+    cursor_field: List[str] = Field(default_factory=list, alias="cursorField")
+    primary_key: List[List[str]] = Field(default_factory=list, alias="primaryKey")
+    destination_sync_mode: AirbyteSyncMode = Field(
+        default=AirbyteSyncMode.OVERWRITE, alias="destinationSyncMode"
+    )
     selected: bool = True
     stream_schema: Optional[AirbyteStreamSchema] = Field(None, alias="schema")
 
-    class Config:
-        """Pydantic configuration."""
-
-        fields = {
-            "name": {"alias": "name"},
-            "namespace": {"alias": "namespace"},
-            "sync_mode": {"alias": "syncMode"},
-            "destination_sync_mode": {"alias": "destinationSyncMode"},
-            "cursor_field": {"alias": "cursorField"},
-            "primary_key": {"alias": "primaryKey"},
-        }
-
 
 class FieldSelection(BaseModel):
     """Mock class for field selection in tests."""
@@ -158,7 +152,7 @@ def test_from_dict(self):
             "securityUpdates": True,
             "displaySetupWizard": False,
         }
-        workspace = AirbyteWorkspace.parse_obj(data)
+        workspace = AirbyteWorkspace.model_validate(data)
         assert workspace.workspace_id == "test-workspace-id"
         assert workspace.name == "Test Workspace"
         assert not hasattr(workspace, "slug")
@@ -170,7 +164,7 @@ def test_from_dict(self):
 
     def test_missing_required_fields(self):
         with pytest.raises(ValidationError):
-            AirbyteWorkspace.parse_obj({})
+            AirbyteWorkspace.model_validate({})
 
     def test_extra_fields_ignored(self):
         data = {
@@ -179,7 +173,7 @@ def test_extra_fields_ignored(self):
             "slug": "test-workspace",
             "extra_field": "extra value",
         }
-        workspace = AirbyteWorkspace.parse_obj(data)
+        workspace = AirbyteWorkspace.model_validate(data)
         assert not hasattr(workspace, "extra_field")
 
 
@@ -211,7 +205,7 @@ def test_from_dict(self):
             "workspaceId": "test-workspace-id",
             "configuration": {"host": "localhost", "port": 5432},
         }
-        source = AirbyteDataSource.parse_obj(data)
+        source = AirbyteDataSource.model_validate(data)
         assert source.source_id == "test-source-id"
         assert source.name == "Test Source"
         assert source.source_type == "postgres"
@@ -222,7 +216,7 @@ def test_from_dict(self):
 
     def test_missing_required_fields(self):
         with pytest.raises(ValidationError):
-            AirbyteDataSource.parse_obj({})
+            AirbyteDataSource.model_validate({})
 
 
 class TestAirbyteDestination:
@@ -253,7 +247,7 @@ def test_from_dict(self):
             "workspaceId": "test-workspace-id",
             "configuration": {"host": "localhost", "port": 5432},
         }
-        destination = AirbyteDestination.parse_obj(data)
+        destination = AirbyteDestination.model_validate(data)
         assert destination.destination_id == "test-destination-id"
         assert destination.name == "Test Destination"
         assert destination.destination_type == "postgres"
@@ -264,7 +258,7 @@ def test_from_dict(self):
 
     def test_missing_required_fields(self):
         with pytest.raises(ValidationError):
-            AirbyteDestination.parse_obj({})
+            AirbyteDestination.model_validate({})
 
 
 class TestAirbyteConnection:
@@ -367,7 +361,7 @@ def test_from_dict(self):
                 "memory_limit": "",
             },
         }
-        connection = AirbyteConnection.parse_obj(data)
+        connection = AirbyteConnection.model_validate(data)
         assert connection.connection_id == "test-connection-id"
         assert connection.name == "Test Connection"
         assert connection.source_id == "test-source-id"
@@ -378,7 +372,7 @@ def test_from_dict(self):
 
     def test_missing_required_fields(self):
         with pytest.raises(ValidationError):
-            AirbyteConnection.parse_obj({})
+            AirbyteConnection.model_validate({})
 
 
 if __name__ == "__main__":

metadata-ingestion/tests/unit/airbyte/test_source_mapping.py

@@ -522,13 +522,13 @@ def test_fetch_streams_for_source(self):
             property_fields=["id", "customer_id", "order_date", "total"],
         )
 
-        # Mock AirbyteStreamDetails.parse_obj to handle any input and return our predefined objects
+        # Mock AirbyteStreamDetails.model_validate to handle any input and return our predefined objects
         with patch(
-            "datahub.ingestion.source.airbyte.models.AirbyteStreamDetails.parse_obj"
-        ) as mock_parse_obj:
+            "datahub.ingestion.source.airbyte.models.AirbyteStreamDetails.model_validate"
+        ) as mock_model_validate:
             # Configure the mock to return our predefined objects regardless of input
             # This bypasses the bug in the implementation where it tries to create a set with a list
-            mock_parse_obj.side_effect = lambda x: (
+            mock_model_validate.side_effect = lambda x: (
                 customer_stream_details
                 if "customers" in str(x)
                 else order_stream_details