Move Google provider to new provider structure (#46344)

* init run on google and fix mypy issues

* fix tests

* replace providers.tests.system.google with providers.google.tests.google

* add example dag's path for new structure

* fix paths in docs

* fix import path for failing test in test_dataprep_system

* fix DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID import

* update google test paths

* replace providers/tests/google with providers/google/tests/provider_tests/google

* remove test_get_package_extras_for_old_providers

* fix test_folders_for_parallel_test_types test

* fix test_pytest_args_for_regular_test_types

* replace providers/tests/system/google/ with providers/google/tests/system/google

* update example_dirs in example_not_excluded_dags

* fix glob path in ExampleCoverageTest.example_paths

* fix providers_prefix

* fix serialization tests

* fix

* add filterwarnings to test_bigquery and test_kubernetes_engine modules

* add explicit info logging to test_dataflow.py

* add explicit info logging to test_datafusion.py

* add explicit info logging to test_datafusion.py

* add explicit info logging to failing caplog tests

* enable info logging

* add tmp.tar.gz

Author: rawwar

.github/boring-cyborg.yml

@@ -171,10 +171,7 @@ labelPRBasedOnFilePath:
     - providers/github/**
 
   provider:google:
-    - providers/src/airflow/providers/google/**/*
-    - docs/apache-airflow-providers-google/**/*
-    - providers/tests/google/**/*
-    - providers/tests/system/google/**/*
+    - providers/google/**
 
   provider:grpc:
     - providers/grpc/**

.pre-commit-config.yaml

@@ -312,7 +312,7 @@ repos:
         exclude: material-icons\.css$|^images/.*$|^RELEASE_NOTES\.txt$|^.*package-lock\.json$|^.*/kinglear\.txt$|^.*pnpm-lock\.yaml$
         args:
           - --ignore-words=docs/spelling_wordlist.txt
-          - --skip=providers/src/airflow/providers/*/*.rst,airflow/www/*.log,docs/*/commits.rst,providers/*/docs/commits.rst,providers/*/*/docs/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md
+          - --skip=providers/src/airflow/providers/*/*.rst,providers/*/docs/changelog.rst,airflow/www/*.log,docs/*/commits.rst,providers/*/docs/commits.rst,providers/*/*/docs/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md
           - --exclude-file=.codespellignorelines
   - repo: https://github.com/woodruffw/zizmor-pre-commit
     rev: v1.0.0
@@ -636,10 +636,10 @@ repos:
           ^providers/exasol/src/airflow/providers/exasol/hooks/exasol.py$|
           ^providers/fab/src/airflow/providers/fab/auth_manager/security_manager/|
           ^providers/fab/src/airflow/providers/fab/www/static/css/bootstrap-theme.css$|
-          ^providers/src/airflow/providers/google/cloud/hooks/bigquery.py$|
-          ^providers/src/airflow/providers/google/cloud/operators/cloud_build.py$|
-          ^providers/src/airflow/providers/google/cloud/operators/dataproc.py$|
-          ^providers/src/airflow/providers/google/cloud/operators/mlengine.py$|
+          ^providers/google/src/airflow/providers/google/cloud/hooks/bigquery.py$|
+          ^providers/google/src/airflow/providers/google/cloud/operators/cloud_build.py$|
+          ^providers/google/src/airflow/providers/google/cloud/operators/dataproc.py$|
+          ^providers/google/src/airflow/providers/google/cloud/operators/mlengine.py$|
           ^providers/src/airflow/providers/microsoft/azure/hooks/cosmos.py$|
           ^providers/src/airflow/providers/microsoft/winrm/hooks/winrm.py$|
           ^airflow/www/fab_security/manager.py$|
@@ -663,7 +663,7 @@ repos:
           ^providers/apache/kafka/docs/connections/kafka.rst$|
           ^providers/apache/spark/docs/decorators/pyspark.rst$|
           ^providers/fab/docs/auth-manager/webserver-authentication.rst$|
-          ^docs/apache-airflow-providers-google/operators/cloud/kubernetes_engine.rst$|
+          ^providers/google/docs/operators/cloud/kubernetes_engine.rst$|
           ^docs/apache-airflow-providers-microsoft-azure/connections/azure_cosmos.rst$|
           ^docs/apache-airflow-providers-cncf-kubernetes/operators.rst$|
           ^docs/conf.py$|

contributing-docs/testing/system_tests.rst

@@ -90,7 +90,7 @@ For providers:
 
 .. code-block:: bash
 
-  pytest --system providers/tests/system/google/cloud/bigquery/example_bigquery_queries.py
+  pytest --system providers/google/tests/system/google/cloud/bigquery/example_bigquery_queries.py
 
 
 Running via Breeze

dev/breeze/src/airflow_breeze/utils/run_tests.py

@@ -217,8 +217,8 @@ def get_excluded_provider_args(python_version: str) -> list[str]:
     "tests/dags_with_system_exit",
     "tests/dags_corrupted",
     "tests/dags",
-    "providers/tests/system/google/cloud/dataproc/resources",
-    "providers/tests/system/google/cloud/gcs/resources",
+    "providers/google/tests/system/google/cloud/dataproc/resources",
+    "providers/google/tests/system/google/cloud/gcs/resources",
 ]
 
 

dev/breeze/tests/test_packages.py

@@ -35,9 +35,7 @@
     get_min_airflow_version,
     get_old_documentation_package_path,
     get_old_source_providers_package_path,
-    get_package_extras_for_old_providers,
     get_pip_package_name,
-    get_provider_details,
     get_provider_info_dict,
     get_provider_jinja_context,
     get_provider_requirements,
@@ -238,122 +236,6 @@ def test_get_install_requirements(provider: str, version_suffix: str, expected:
     assert actual.strip() == expected.strip()
 
 
-# TODO(potiuk) - remove when all providers are new-style
-@pytest.mark.parametrize(
-    "version_suffix, expected",
-    [
-        pytest.param(
-            "",
-            {
-                "amazon": ["apache-airflow-providers-amazon>=2.6.0"],
-                "apache.beam": ["apache-airflow-providers-apache-beam", "apache-beam[gcp]"],
-                "apache.cassandra": ["apache-airflow-providers-apache-cassandra"],
-                "cncf.kubernetes": ["apache-airflow-providers-cncf-kubernetes>=10.1.0"],
-                "facebook": ["apache-airflow-providers-facebook>=2.2.0"],
-                "leveldb": ["plyvel>=1.5.1"],
-                "microsoft.azure": ["apache-airflow-providers-microsoft-azure"],
-                "microsoft.mssql": ["apache-airflow-providers-microsoft-mssql"],
-                "mysql": ["apache-airflow-providers-mysql"],
-                "openlineage": ["apache-airflow-providers-openlineage"],
-                "oracle": ["apache-airflow-providers-oracle>=3.1.0"],
-                "postgres": ["apache-airflow-providers-postgres"],
-                "presto": ["apache-airflow-providers-presto"],
-                "salesforce": ["apache-airflow-providers-salesforce"],
-                "sftp": ["apache-airflow-providers-sftp"],
-                "ssh": ["apache-airflow-providers-ssh"],
-                "trino": ["apache-airflow-providers-trino"],
-            },
-            id="No suffix",
-        ),
-        pytest.param(
-            "dev0",
-            {
-                "amazon": ["apache-airflow-providers-amazon>=2.6.0.dev0"],
-                "apache.beam": ["apache-airflow-providers-apache-beam", "apache-beam[gcp]"],
-                "apache.cassandra": ["apache-airflow-providers-apache-cassandra"],
-                "cncf.kubernetes": ["apache-airflow-providers-cncf-kubernetes>=10.1.0.dev0"],
-                "facebook": ["apache-airflow-providers-facebook>=2.2.0.dev0"],
-                "leveldb": ["plyvel>=1.5.1"],
-                "microsoft.azure": ["apache-airflow-providers-microsoft-azure"],
-                "microsoft.mssql": ["apache-airflow-providers-microsoft-mssql"],
-                "mysql": ["apache-airflow-providers-mysql"],
-                "openlineage": ["apache-airflow-providers-openlineage"],
-                "oracle": ["apache-airflow-providers-oracle>=3.1.0.dev0"],
-                "postgres": ["apache-airflow-providers-postgres"],
-                "presto": ["apache-airflow-providers-presto"],
-                "salesforce": ["apache-airflow-providers-salesforce"],
-                "sftp": ["apache-airflow-providers-sftp"],
-                "ssh": ["apache-airflow-providers-ssh"],
-                "trino": ["apache-airflow-providers-trino"],
-            },
-            id="With dev0 suffix",
-        ),
-        pytest.param(
-            "beta0",
-            {
-                "amazon": ["apache-airflow-providers-amazon>=2.6.0b0"],
-                "apache.beam": ["apache-airflow-providers-apache-beam", "apache-beam[gcp]"],
-                "apache.cassandra": ["apache-airflow-providers-apache-cassandra"],
-                "cncf.kubernetes": ["apache-airflow-providers-cncf-kubernetes>=10.1.0b0"],
-                "facebook": ["apache-airflow-providers-facebook>=2.2.0b0"],
-                "leveldb": ["plyvel>=1.5.1"],
-                "microsoft.azure": ["apache-airflow-providers-microsoft-azure"],
-                "microsoft.mssql": ["apache-airflow-providers-microsoft-mssql"],
-                "mysql": ["apache-airflow-providers-mysql"],
-                "openlineage": ["apache-airflow-providers-openlineage"],
-                "oracle": ["apache-airflow-providers-oracle>=3.1.0b0"],
-                "postgres": ["apache-airflow-providers-postgres"],
-                "presto": ["apache-airflow-providers-presto"],
-                "salesforce": ["apache-airflow-providers-salesforce"],
-                "sftp": ["apache-airflow-providers-sftp"],
-                "ssh": ["apache-airflow-providers-ssh"],
-                "trino": ["apache-airflow-providers-trino"],
-            },
-            id="With beta0 suffix normalized automatically to b0 (PEP 440)",
-        ),
-    ],
-)
-def test_get_package_extras_for_old_providers(version_suffix: str, expected: dict[str, list[str]]):
-    actual = get_package_extras_for_old_providers("google", version_suffix=version_suffix)
-    expected_as_list: list[str] = []
-    for package, extras in expected.items():
-        expected_as_list.append(f'"{package}" = [')
-        for extra in extras:
-            expected_as_list.append(f'     "{extra}",')
-        expected_as_list.append("]")
-    expected_as_str = "\n".join(expected_as_list)
-    assert actual == expected_as_str
-
-
-# TODO(potiuk) - remove when all providers are new-style
-def test_get_new_provider_details():
-    provider_details = get_provider_details("airbyte")
-    assert provider_details.provider_id == "airbyte"
-    assert provider_details.full_package_name == "airflow.providers.airbyte"
-    assert provider_details.pypi_package_name == "apache-airflow-providers-airbyte"
-    assert provider_details.root_provider_path == AIRFLOW_SOURCES_ROOT.joinpath(
-        "providers",
-        "airbyte",
-    )
-    assert provider_details.base_provider_package_path == AIRFLOW_SOURCES_ROOT.joinpath(
-        "providers",
-        "airbyte",
-        "src",
-        "airflow",
-        "providers",
-        "airbyte",
-    )
-    assert provider_details.documentation_provider_package_path == AIRFLOW_SOURCES_ROOT.joinpath(
-        "providers", "airbyte", "docs"
-    )
-    assert "Airbyte" in provider_details.provider_description
-    assert len(provider_details.versions) > 11
-    assert provider_details.excluded_python_versions == []
-    assert provider_details.plugins == []
-    assert provider_details.changelog_path == provider_details.root_provider_path / "docs" / "changelog.rst"
-    assert not provider_details.removed
-
-
 @pytest.mark.parametrize(
     "provider_id, pip_package_name",
     [

dev/breeze/tests/test_pytest_args_for_test_types.py

@@ -95,16 +95,23 @@ def _all_new_providers() -> list[str]:
         (
             GroupOfTests.PROVIDERS,
             "Providers[amazon,google,apache.hive]",
-            ["providers/tests/amazon", "providers/tests/google", "providers/apache/hive/tests"],
+            [
+                "providers/tests/amazon",
+                "providers/google/tests",
+                "providers/apache/hive/tests",
+            ],
         ),
         (
             GroupOfTests.PROVIDERS,
             "Providers[-amazon,google,microsoft.azure]",
             [
-                *[f"providers/{provider}/tests" for provider in _all_new_providers()],
+                *[
+                    f"providers/{provider}/tests"
+                    for provider in _all_new_providers()
+                    if provider not in ["amazon", "google", "microsoft/azure"]
+                ],
                 "providers/tests",
                 "--ignore=providers/tests/amazon",
-                "--ignore=providers/tests/google",
                 "--ignore=providers/tests/microsoft/azure",
             ],
         ),
@@ -244,23 +251,34 @@ def test_pytest_args_for_missing_provider():
             "Providers[amazon] Providers[google]",
             [
                 "providers/tests/amazon",
-                "providers/tests/google",
+                "providers/google/tests",
             ],
         ),
         (
             GroupOfTests.PROVIDERS,
             "Providers[-amazon,google]",
             [
-                *[f"providers/{provider}/tests" for provider in _all_new_providers()],
+                *[
+                    f"providers/{provider}/tests"
+                    for provider in _all_new_providers()
+                    if provider not in ["amazon", "google"]
+                ],
                 "providers/tests",
             ],
         ),
         (
             GroupOfTests.PROVIDERS,
             "Providers[-amazon,google] Providers[amazon] Providers[google]",
             [
-                *[f"providers/{provider}/tests" for provider in _all_new_providers()],
+                *[
+                    f"providers/{provider}/tests"
+                    for provider in _all_new_providers()
+                    if provider not in ["amazon", "google"]
+                ],
                 "providers/tests",
+                *[
+                    "providers/google/tests"
+                ],  # Once amazon is migrated to the new structure, amazon needs to be added to the list here.
             ],
         ),
         (

dev/breeze/tests/test_selective_checks.py

@@ -1384,7 +1384,7 @@ def test_expected_output_full_tests_needed(
         pytest.param(
             (
                 "chart/aaaa.txt",
-                "providers/tests/google/file.py",
+                "providers/google/tests/provider_tests/google/file.py",
             ),
             {
                 "all-python-versions": "['3.9']",
@@ -1411,7 +1411,7 @@ def test_expected_output_full_tests_needed(
             (
                 "airflow/cli/test.py",
                 "chart/aaaa.txt",
-                "providers/tests/google/file.py",
+                "providers/google/tests/provider_tests/google/file.py",
             ),
             {
                 "all-python-versions": "['3.9']",
@@ -1436,7 +1436,7 @@ def test_expected_output_full_tests_needed(
         pytest.param(
             (
                 "airflow/file.py",
-                "providers/tests/google/file.py",
+                "providers/google/tests/provider_tests/google/file.py",
             ),
             {
                 "all-python-versions": "['3.9']",
@@ -1614,7 +1614,7 @@ def test_expected_output_push(
             (
                 "airflow/cli/test.py",
                 "chart/aaaa.txt",
-                "providers/tests/google/file.py",
+                "providers/google/tests/provider_tests/google/file.py",
             ),
             {
                 "selected-providers-list-as-string": "amazon apache.beam apache.cassandra "

docs/.gitignore

@@ -34,6 +34,7 @@ apache-airflow-providers-fab
 apache-airflow-providers-facebook
 apache-airflow-providers-ftp
 apache-airflow-providers-github
+apache-airflow-providers-google
 apache-airflow-providers-grpc
 apache-airflow-providers-http
 apache-airflow-providers-jdbc