Run postcommits.

Author: claudevdm

.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 3
+  "modification": 4
 }

build.gradle.kts

@@ -502,40 +502,26 @@ tasks.register("pythonFormatterPreCommit") {
 }
 
 tasks.register("python39PostCommit") {
-  dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitIT")
+  // dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py39:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py39:hdfsIntegrationTest")
-  dependsOn(":sdks:python:test-suites:direct:py39:azureIntegrationTest")
-  dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39")
+  // dependsOn(":sdks:python:test-suites:direct:py39:hdfsIntegrationTest")
+  // dependsOn(":sdks:python:test-suites:direct:py39:azureIntegrationTest")
+  // dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39")
   // TODO (https://github.com/apache/beam/issues/23966)
   // Move this to Python 3.10 test suite once tfx-bsl has python 3.10 wheel.
-  dependsOn(":sdks:python:test-suites:direct:py39:inferencePostCommitIT")
+  // dependsOn(":sdks:python:test-suites:direct:py39:inferencePostCommitIT")
 }
 
 tasks.register("python310PostCommit") {
-  dependsOn(":sdks:python:test-suites:dataflow:py310:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py310:postCommitIT")
-  dependsOn(":sdks:python:test-suites:portable:py310:postCommitPy310")
-  // TODO: https://github.com/apache/beam/issues/22651
-  // The default container uses Python 3.10. The goal here is to
-  // duild Docker images for TensorRT tests during run time for python versions
-  // other than 3.10 and add these tests in other python postcommit suites.
-  dependsOn(":sdks:python:test-suites:dataflow:py310:inferencePostCommitIT")
 }
 
 tasks.register("python311PostCommit") {
-  dependsOn(":sdks:python:test-suites:dataflow:py311:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py311:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py311:hdfsIntegrationTest")
-  dependsOn(":sdks:python:test-suites:portable:py311:postCommitPy311")
 }
 
 tasks.register("python312PostCommit") {
-  dependsOn(":sdks:python:test-suites:dataflow:py312:postCommitIT")
   dependsOn(":sdks:python:test-suites:direct:py312:postCommitIT")
-  dependsOn(":sdks:python:test-suites:direct:py312:hdfsIntegrationTest")
-  dependsOn(":sdks:python:test-suites:portable:py312:postCommitPy312")
-  dependsOn(":sdks:python:test-suites:dataflow:py312:inferencePostCommitITPy312")
 }
 
 tasks.register("portablePythonPreCommit") {

sdks/python/apache_beam/ml/rag/ingestion/alloydb_it_test.py

@@ -26,6 +26,7 @@
 from typing import NamedTuple
 
 import psycopg2
+import pytest
 
 import apache_beam as beam
 from apache_beam.coders import registry
@@ -142,6 +143,11 @@ def key_on_id(chunk):
   return (int(chunk.id.split('_')[1]), chunk)
 
 
+@pytest.mark.uses_gcp_java_expansion_service
+@unittest.skipUnless(
+    os.environ.get('EXPANSION_JARS'),
+    "EXPANSION_JARS environment var is not provided, "
+    "indicating that jars have not been built")
 class AlloyDBVectorWriterConfigTest(unittest.TestCase):
   ALLOYDB_TABLE_PREFIX = 'python_rag_alloydb_'
 

sdks/python/test-suites/dataflow/common.gradle

@@ -128,7 +128,7 @@ task postCommitIT {
   dependsOn 'initializeForDataflowJob'
 
   doLast {
-    def testOpts = basicPytestOpts + ["--numprocesses=8", "--dist=loadfile"]
+    def testOpts = basicPytestOpts + ["--numprocesses=8", "--dist=loadfile", "apache_beam/ml/rag/"]
     def argMap = [
         "test_opts": testOpts,
         "sdk_location": project.ext.sdkLocation,

sdks/python/test-suites/direct/common.gradle

@@ -35,13 +35,7 @@ tasks.register("postCommitIT") {
   // Run IT tests with TestDirectRunner in batch in Python 3.
   doLast {
     def batchTests = [
-        "apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it",
-        "apache_beam/io/gcp/pubsub_integration_test.py::PubSubIntegrationTest",
-        "apache_beam/io/gcp/big_query_query_to_table_it_test.py::BigQueryQueryToTableIT",
-        "apache_beam/io/gcp/bigquery_io_read_it_test.py",
-        "apache_beam/io/gcp/bigquery_read_it_test.py",
-        "apache_beam/io/gcp/bigquery_write_it_test.py",
-        "apache_beam/io/gcp/datastore/v1new/datastore_write_it_test.py",
+        "apache_beam/ml/rag/",
     ]
     def testOpts = basicTestOpts + ["${batchTests.join(' ')}"]
     def argMap = ["runner": "TestDirectRunner",
@@ -136,13 +130,7 @@ tasks.register("directRunnerIT") {
   // Run IT tests with TestDirectRunner in batch.
   doLast {
     def tests = [
-        "apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it",
-        "apache_beam/io/gcp/pubsub_integration_test.py::PubSubIntegrationTest",
-        "apache_beam/io/gcp/big_query_query_to_table_it_test.py::BigQueryQueryToTableIT",
-        "apache_beam/io/gcp/bigquery_io_read_it_test.py",
-        "apache_beam/io/gcp/bigquery_read_it_test.py",
-        "apache_beam/io/gcp/bigquery_write_it_test.py",
-        "apache_beam/io/gcp/datastore/v1new/datastore_write_it_test.py",
+        "apache_beam/ml/rag/",
     ]
     def batchTestOpts = basicTestOpts + ["${tests.join(' ')}"]
     def argMap = ["runner": "TestDirectRunner",
@@ -159,13 +147,7 @@ tasks.register("directRunnerIT") {
   // Run IT tests with TestDirectRunner in streaming.
   doLast {
     def tests = [
-        "apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it",
-        "apache_beam/io/gcp/pubsub_integration_test.py::PubSubIntegrationTest",
-        "apache_beam/io/gcp/bigquery_test.py::BigQueryStreamingInsertTransformIntegrationTests::test_multiple_destinations_transform",
-        "apache_beam/io/gcp/bigquery_test.py::PubSubBigQueryIT",
-        "apache_beam/io/gcp/bigquery_file_loads_test.py::BigQueryFileLoadsIT::test_bqfl_streaming",
-        "apache_beam/io/gcp/bigquery_file_loads_test.py::BigQueryFileLoadsIT::test_bqfl_streaming_with_dynamic_destinations",
-        "apache_beam/io/gcp/bigquery_file_loads_test.py::BigQueryFileLoadsIT::test_bqfl_streaming_with_copy_jobs",
+        "apache_beam/ml/rag/",
     ]
     def streamingTestOpts = basicTestOpts + ["${tests.join(' ')}"]
     def argMap = ["runner": "TestDirectRunner",
@@ -454,6 +436,7 @@ project(":sdks:python:test-suites:xlang").ext.xlangTasks.each { taskMetadata ->
         "--timeout=4500", // timeout of whole command execution
         "--color=yes", // console color
         "--log-cli-level=INFO" //log level info
+        "apache_beam/ml/rag/ingestion/alloydb_it_test.py"
       ],
       additionalDeps: taskMetadata.additionalDeps,
       additionalEnvs: taskMetadata.additionalEnvs

sdks/python/tox.ini

@@ -106,7 +106,6 @@ commands =
 [testenv:py{39,310,311}-ml]
 # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
 setenv =
-passenv = ALLOYDB_PASSWORD
 extras = test,gcp,dataframe,ml_test
 commands =
   # Log tensorflow version for debugging
@@ -117,7 +116,6 @@ commands =
 # many packages do not support py3.12
 # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
 setenv =
-passenv = ALLOYDB_PASSWORD
 extras = test,gcp,dataframe,p312_ml_test
 commands =
   # Log tensorflow version for debugging