This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new cbdf30b52ae [YAML]: fix postcommit oracle bug and reorganize
postcommit tests (#35191)
cbdf30b52ae is described below
commit cbdf30b52aea5db5cc477c8bfd747396d3f1e245
Author: Derrick Williams <[email protected]>
AuthorDate: Thu Jun 12 14:49:02 2025 -0400
[YAML]: fix postcommit oracle bug and reorganize postcommit tests (#35191)
* fix postcommit oracle test
* add revision
* switch to hosted runner to try with kafka test
* add extended timeout
* upgrade to 4.10 testcontainers
* switch out to redpanda for kafka
* remove redpandacontainer
* tmp comment
* add postgres comment
* revert to old kafkaContainer
* revert commented out code and revert testcontainer version change
* change mysql image version
* revert image change
* revert image change again :)
* add comments to mysql again
* shift post commit files to different folders
* rename to Data version
* add databases version
* add messaging version
* update readme for three post commits
* update gradle with new post commits
* fix job names
* uncomment fixture on mysql
* switch back to one workflow and update readme as such
* remove old workflow files
* update order and remove comment
* update gradle with parameterized options
* update gradle command calls to correct location
* update workflow to three jobs explicit
* add back Bigquery deselect
* fix mysql teardown error
* Simplify down to one from three explicit jobs
* remove tab
* remove Data
* fix parsing parameters
---
.../beam_PostCommit_Yaml_Xlang_Direct.json | 2 +-
.../beam_PostCommit_Yaml_Xlang_Direct.yml | 12 +++++----
.../yaml/extended_tests/{ => data}/enrichment.yaml | 0
.../yaml/extended_tests/{ => data}/tfrecord.yaml | 0
.../extended_tests/{ => databases}/bigquery.yaml | 0
.../yaml/extended_tests/{ => databases}/jdbc.yaml | 0
.../yaml/extended_tests/{ => databases}/mysql.yaml | 0
.../extended_tests/{ => databases}/oracle.yaml | 0
.../extended_tests/{ => databases}/postgres.yaml | 0
.../extended_tests/{ => databases}/spanner.yaml | 0
.../extended_tests/{ => databases}/sqlserver.yaml | 0
.../yaml/extended_tests/{ => messaging}/kafka.yaml | 0
.../extended_tests/{ => messaging}/pubsub.yaml | 0
sdks/python/apache_beam/yaml/integration_tests.py | 12 ++++-----
sdks/python/build.gradle | 29 ++++++++++++++++++----
sdks/python/setup.py | 3 ++-
16 files changed, 40 insertions(+), 18 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json
b/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json
index a975cd1cd10..541dc4ea8e8 100644
--- a/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json
+++ b/.github/trigger_files/beam_PostCommit_Yaml_Xlang_Direct.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run",
- "revision": 1
+ "revision": 2
}
diff --git a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml
b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml
index 8ec3c2bc7aa..1f3d5147566 100644
--- a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml
+++ b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml
@@ -54,21 +54,22 @@ jobs:
github.event_name == 'workflow_dispatch' ||
github.event_name == 'pull_request_target' ||
(github.event_name == 'schedule' && github.repository == 'apache/beam')
- runs-on: [self-hosted, ubuntu-20.04, main]
+ runs-on: ubuntu-latest
timeout-minutes: 100
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
strategy:
matrix:
job_name: ["beam_PostCommit_Yaml_Xlang_Direct"]
job_phrase: ["Run Yaml_Xlang_Direct PostCommit"]
+ test_set: ["data", "databases", "messaging"]
steps:
- uses: actions/checkout@v4
- name: Setup repository
uses: ./.github/actions/setup-action
with:
- comment_phrase: ${{ matrix.job_phrase }}
+ comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.test_set }}
github_token: ${{ secrets.GITHUB_TOKEN }}
- github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+ github_job: ${{ matrix.job_name }} ${{ matrix.test_set }} (${{
matrix.job_phrase }} ${{ matrix.test_set }})
- name: Setup environment
uses: ./.github/actions/setup-environment-action
with:
@@ -79,7 +80,7 @@ jobs:
- name: run PostCommit Yaml Xlang Direct script
uses: ./.github/actions/gradle-command-self-hosted-action
with:
- gradle-command: :sdks:python:postCommitYamlIntegrationTests
-PbeamPythonExtra=ml_test
+ gradle-command: :sdks:python:postCommitYamlIntegrationTests
-PyamlTestSet=${{ matrix.test_set }} -PbeamPythonExtra=ml_test
- name: Archive Python Test Results
uses: actions/upload-artifact@v4
if: failure()
@@ -93,4 +94,5 @@ jobs:
commit: '${{ env.prsha || env.GITHUB_SHA }}'
comment_mode: ${{ github.event_name == 'issue_comment' && 'always'
|| 'off' }}
files: '**/pytest*.xml'
- large_files: true
\ No newline at end of file
+ large_files: true
+
diff --git a/sdks/python/apache_beam/yaml/extended_tests/enrichment.yaml
b/sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml
similarity index 100%
rename from sdks/python/apache_beam/yaml/extended_tests/enrichment.yaml
rename to sdks/python/apache_beam/yaml/extended_tests/data/enrichment.yaml
diff --git a/sdks/python/apache_beam/yaml/extended_tests/tfrecord.yaml
b/sdks/python/apache_beam/yaml/extended_tests/data/tfrecord.yaml
similarity index 100%
rename from sdks/python/apache_beam/yaml/extended_tests/tfrecord.yaml
rename to sdks/python/apache_beam/yaml/extended_tests/data/tfrecord.yaml
diff --git a/sdks/python/apache_beam/yaml/extended_tests/bigquery.yaml
b/sdks/python/apache_beam/yaml/extended_tests/databases/bigquery.yaml
similarity index 100%
rename from sdks/python/apache_beam/yaml/extended_tests/bigquery.yaml
rename to sdks/python/apache_beam/yaml/extended_tests/databases/bigquery.yaml
diff --git a/sdks/python/apache_beam/yaml/extended_tests/jdbc.yaml
b/sdks/python/apache_beam/yaml/extended_tests/databases/jdbc.yaml
similarity index 100%
rename from sdks/python/apache_beam/yaml/extended_tests/jdbc.yaml
rename to sdks/python/apache_beam/yaml/extended_tests/databases/jdbc.yaml
diff --git a/sdks/python/apache_beam/yaml/extended_tests/mysql.yaml
b/sdks/python/apache_beam/yaml/extended_tests/databases/mysql.yaml
similarity index 100%
rename from sdks/python/apache_beam/yaml/extended_tests/mysql.yaml
rename to sdks/python/apache_beam/yaml/extended_tests/databases/mysql.yaml
diff --git a/sdks/python/apache_beam/yaml/extended_tests/oracle.yaml
b/sdks/python/apache_beam/yaml/extended_tests/databases/oracle.yaml
similarity index 100%
rename from sdks/python/apache_beam/yaml/extended_tests/oracle.yaml
rename to sdks/python/apache_beam/yaml/extended_tests/databases/oracle.yaml
diff --git a/sdks/python/apache_beam/yaml/extended_tests/postgres.yaml
b/sdks/python/apache_beam/yaml/extended_tests/databases/postgres.yaml
similarity index 100%
rename from sdks/python/apache_beam/yaml/extended_tests/postgres.yaml
rename to sdks/python/apache_beam/yaml/extended_tests/databases/postgres.yaml
diff --git a/sdks/python/apache_beam/yaml/extended_tests/spanner.yaml
b/sdks/python/apache_beam/yaml/extended_tests/databases/spanner.yaml
similarity index 100%
rename from sdks/python/apache_beam/yaml/extended_tests/spanner.yaml
rename to sdks/python/apache_beam/yaml/extended_tests/databases/spanner.yaml
diff --git a/sdks/python/apache_beam/yaml/extended_tests/sqlserver.yaml
b/sdks/python/apache_beam/yaml/extended_tests/databases/sqlserver.yaml
similarity index 100%
rename from sdks/python/apache_beam/yaml/extended_tests/sqlserver.yaml
rename to sdks/python/apache_beam/yaml/extended_tests/databases/sqlserver.yaml
diff --git a/sdks/python/apache_beam/yaml/extended_tests/kafka.yaml
b/sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml
similarity index 100%
rename from sdks/python/apache_beam/yaml/extended_tests/kafka.yaml
rename to sdks/python/apache_beam/yaml/extended_tests/messaging/kafka.yaml
diff --git a/sdks/python/apache_beam/yaml/extended_tests/pubsub.yaml
b/sdks/python/apache_beam/yaml/extended_tests/messaging/pubsub.yaml
similarity index 100%
rename from sdks/python/apache_beam/yaml/extended_tests/pubsub.yaml
rename to sdks/python/apache_beam/yaml/extended_tests/messaging/pubsub.yaml
diff --git a/sdks/python/apache_beam/yaml/integration_tests.py
b/sdks/python/apache_beam/yaml/integration_tests.py
index f6c60ae5f12..6824105737a 100644
--- a/sdks/python/apache_beam/yaml/integration_tests.py
+++ b/sdks/python/apache_beam/yaml/integration_tests.py
@@ -228,7 +228,7 @@ def temp_mysql_database():
with the MySQL database during setup.
Exception: Any other exception encountered during the setup process.
"""
- with MySqlContainer() as mysql_container:
+ with MySqlContainer(init=True) as mysql_container:
try:
# Make connection to temp database and create tmp table
engine = sqlalchemy.create_engine(mysql_container.get_connection_url())
@@ -440,12 +440,12 @@ def temp_kafka_server():
Exception: If there's an error starting the Kafka container or
interacting with the temporary Kafka server.
"""
- try:
- with KafkaContainer() as kafka_container:
+ with KafkaContainer() as kafka_container:
+ try:
yield kafka_container.get_bootstrap_server()
- except Exception as err:
- logging.error("Error interacting with temporary Kakfa Server: %s", err)
- raise err
+ except Exception as err:
+ logging.error("Error interacting with temporary Kakfa Server: %s", err)
+ raise err
@contextlib.contextmanager
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index accacb1953b..c8f02262d3f 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -138,13 +138,13 @@ tasks.register("yamlIntegrationTests") {
doLast {
exec {
executable 'sh'
- args '-c', "${envdir}/bin/pytest -v
apache_beam/yaml/integration_tests.py --deselect
apache_beam/yaml/integration_tests.py::BigqueryTest::test_ReadFromBigQuery_ExternalJavaProvider_0"
+ args '-c', "${envdir}/bin/pytest -v
apache_beam/yaml/integration_tests.py"
}
}
}
tasks.register("postCommitYamlIntegrationTests") {
- description "Runs postcommit integration tests for yaml pipelines."
+ description "Runs postcommit integration tests for yaml pipelines -
parameterized by yamlTestSet."
dependsOn installGcpTest
// Need to build all expansion services referenced in apache_beam/yaml/*.*
@@ -155,9 +155,28 @@ tasks.register("postCommitYamlIntegrationTests") {
dependsOn ":sdks:java:io:google-cloud-platform:expansion-service:build"
doLast {
- exec {
- executable 'sh'
- args '-c', "${envdir}/bin/pytest -v
apache_beam/yaml/integration_tests.py --deselect
apache_beam/yaml/integration_tests.py::BigqueryTest::test_ReadFromBigQuery_ExternalJavaProvider_0
--test_files_dir='extended_tests'"
+ def testSetInput = project.findProperty('yamlTestSet') ?:
'data,databases,messaging'
+ def testSetsToRun = testSetInput.tokenize(',').collect { it.trim()
}.findAll { !it.isEmpty() }
+ testSetsToRun.each { currentTestSet ->
+ def test_files_dir
+
+ switch (currentTestSet) {
+ case 'data':
+ test_files_dir = 'extended_tests/data'
+ break
+ case 'databases':
+ test_files_dir = 'extended_tests/databases'
+ break
+ case 'messaging':
+ test_files_dir = 'extended_tests/messaging'
+ break
+ default:
+ throw StopExecutionException("Unknown yamlTestSet: ${testSet}. Must
be one of 'data', 'databases', or 'messaging'.")
+ }
+ exec {
+ executable 'sh'
+ args '-c', "${envdir}/bin/pytest -v
apache_beam/yaml/integration_tests.py --deselect
apache_beam/yaml/integration_tests.py::BigqueryTest::test_ReadFromBigQuery_ExternalJavaProvider_0
--test_files_dir='${test_files_dir}'"
+ }
}
}
}
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 3750c087f18..630bb7188ed 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -444,7 +444,8 @@ if __name__ == '__main__':
'virtualenv-clone>=0.5,<1.0',
'mysql-connector-python>=9.3.0',
'python-tds>=1.16.1',
- 'sqlalchemy-pytds>=1.0.2'
+ 'sqlalchemy-pytds>=1.0.2',
+ 'oracledb>=3.1.1'
],
'gcp': [
'cachetools>=3.1.0,<6',
