This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 8d7adba289b Fix hdfs integration test (#30458)
8d7adba289b is described below
commit 8d7adba289bac714996585bf871e395242d5d932
Author: Yi Hu <[email protected]>
AuthorDate: Wed Mar 6 13:15:24 2024 -0500
Fix hdfs integration test (#30458)
* Fix hdfs integration test
debug only
skip yaml test
* Add back azureIntegrationTest get removed when teardown Py37 support
---
.github/trigger_files/beam_PostCommit_Python.json | 0
build.gradle.kts | 3 +--
sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml | 4 ++--
sdks/python/test-suites/direct/common.gradle | 6 ++++++
4 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Python.json
b/.github/trigger_files/beam_PostCommit_Python.json
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/build.gradle.kts b/build.gradle.kts
index 788d9a48dc0..ded692677b5 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -504,6 +504,7 @@ tasks.register("python38PostCommit") {
dependsOn(":sdks:python:test-suites:dataflow:py38:postCommitIT")
dependsOn(":sdks:python:test-suites:direct:py38:postCommitIT")
dependsOn(":sdks:python:test-suites:direct:py38:hdfsIntegrationTest")
+ dependsOn(":sdks:python:test-suites:direct:py38:azureIntegrationTest")
dependsOn(":sdks:python:test-suites:portable:py38:postCommitPy38")
// TODO: https://github.com/apache/beam/issues/22651
// The default container uses Python 3.8. The goal here is to
@@ -516,7 +517,6 @@ tasks.register("python38PostCommit") {
tasks.register("python39PostCommit") {
dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitIT")
dependsOn(":sdks:python:test-suites:direct:py39:postCommitIT")
- dependsOn(":sdks:python:test-suites:direct:py39:hdfsIntegrationTest")
dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39")
// TODO (https://github.com/apache/beam/issues/23966)
// Move this to Python 3.10 test suite once tfx-bsl has python 3.10 wheel.
@@ -526,7 +526,6 @@ tasks.register("python39PostCommit") {
tasks.register("python310PostCommit") {
dependsOn(":sdks:python:test-suites:dataflow:py310:postCommitIT")
dependsOn(":sdks:python:test-suites:direct:py310:postCommitIT")
- dependsOn(":sdks:python:test-suites:direct:py310:hdfsIntegrationTest")
dependsOn(":sdks:python:test-suites:portable:py310:postCommitPy310")
}
diff --git
a/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml
b/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml
index 41410028ca3..8248f43bc55 100644
--- a/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml
+++ b/sdks/python/apache_beam/io/hdfs_integration_test/docker-compose.yml
@@ -20,7 +20,7 @@ version: '3'
services:
# HDFS namenode.
namenode:
- image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-namenode:2.8.1
+ image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-namenode:2.10.2
hostname: namenode
environment:
- CLUSTER_NAME=test
@@ -30,7 +30,7 @@ services:
# HDFS datanode.
datanode:
- image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-datanode:2.8.1
+ image: gcr.io/apache-beam-testing/uhopper-hadoop/hadoop-datanode:2.10.2
hostname: datanode
environment:
- CLUSTER_NAME=test
diff --git a/sdks/python/test-suites/direct/common.gradle
b/sdks/python/test-suites/direct/common.gradle
index b5680c2e1e9..c8f4f719f55 100644
--- a/sdks/python/test-suites/direct/common.gradle
+++ b/sdks/python/test-suites/direct/common.gradle
@@ -182,6 +182,9 @@ tasks.register("directRunnerIT") {
}
tasks.register("hdfsIntegrationTest") {
+ // needs generated srcs in sdks/python
+ dependsOn ':sdks:python:sdist'
+
doLast {
exec {
executable 'sh'
@@ -191,6 +194,9 @@ tasks.register("hdfsIntegrationTest") {
}
tasks.register("azureIntegrationTest") {
+ // needs generated srcs in sdks/python
+ dependsOn ':sdks:python:sdist'
+
doLast {
exec {
executable 'sh'
