This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 5d52bb36d3b [SPARK-41486][SQL][TESTS] Upgrade `MySQL` docker image to 8.0.31 to support `ARM64` test 5d52bb36d3b is described below commit 5d52bb36d3ba7e619169289f8a576db5ea10990f Author: Dongjoon Hyun <dongj...@apache.org> AuthorDate: Sun Dec 11 22:23:17 2022 -0800 [SPARK-41486][SQL][TESTS] Upgrade `MySQL` docker image to 8.0.31 to support `ARM64` test ### What changes were proposed in this pull request? This PR aims to upgrade MySQL JDBC Integration Test docker image from 5.7.36 to 8.0.31 to support ARM64 testing. ### Why are the changes needed? Since MySQL 8.0, MySQL provides multi-arch images. After this upgrade, we can run MySQL JDBC Integration Test on ARM64 environment. ![Screenshot 2022-12-11 at 8 31 48 PM](https://user-images.githubusercontent.com/9700541/206961236-c958e6fc-1a2d-40cf-bf53-6f72a312c844.png) ### Does this PR introduce _any_ user-facing change? No, this is a test code PR. ### How was this patch tested? - Pass GitHub Action Ci for AMD64 architecture - Manually run MySQLIntegrationSuites on ARM64 architecture. I manually run this PR on Apple Silicon. ``` $ ./build/sbt -Pdocker-integration-tests "testOnly *MySQLIntegrationSuite" [info] MySQLIntegrationSuite: [info] - SPARK-33034: ALTER TABLE ... add new columns (4 seconds, 766 milliseconds) [info] - SPARK-33034: ALTER TABLE ... drop column (389 milliseconds) [info] - SPARK-33034: ALTER TABLE ... update column type (285 milliseconds) [info] - SPARK-33034: ALTER TABLE ... rename column (206 milliseconds) [info] - SPARK-33034: ALTER TABLE ... update column nullability (167 milliseconds) [info] - CREATE TABLE with table comment (98 milliseconds) [info] - CREATE TABLE with table property (113 milliseconds) [info] - SPARK-36895: Test INDEX Using SQL (559 milliseconds) [info] - SPARK-37038: Test TABLESAMPLE (0 milliseconds) [info] - scan with aggregate push-down: VAR_POP without DISTINCT (3 seconds, 718 milliseconds) [info] - scan with aggregate push-down: VAR_SAMP without DISTINCT (602 milliseconds) [info] - scan with aggregate push-down: STDDEV_POP without DISTINCT (624 milliseconds) [info] - scan with aggregate push-down: STDDEV_SAMP without DISTINCT (390 milliseconds) [info] MySQLIntegrationSuite: [info] Passed: Total 0, Failed 0, Errors 0, Passed 0 [info] No tests to run for mllib / Test / testOnly [info] - Basic test (432 milliseconds) [info] - Numeric types (324 milliseconds) [info] - Date types (871 milliseconds) [info] - String types (302 milliseconds) [info] - Basic write test (1 second, 341 milliseconds) [info] - query JDBC option (334 milliseconds) [info] Run completed in 1 minute, 15 seconds. [info] Total number of tests run: 19 [info] Suites: completed 2, aborted 0 [info] Tests: succeeded 19, failed 0, canceled 0, ignored 0, pending 0 [info] All tests passed. $ ./build/sbt -Pdocker-integration-tests "testOnly *MySQLNamespaceSuite" [info] MySQLNamespaceSuite: [info] - listNamespaces: basic behavior (1 second, 367 milliseconds) [info] - Drop namespace (231 milliseconds) [info] - Create or remove comment of namespace unsupported (186 milliseconds) [info] Run completed in 33 seconds, 456 milliseconds. [info] Total number of tests run: 3 [info] Suites: completed 1, aborted 0 [info] Tests: succeeded 3, failed 0, canceled 0, ignored 0, pending 0 [info] All tests passed. ``` Closes #39029 from dongjoon-hyun/SPARK-41486. Authored-by: Dongjoon Hyun <dongj...@apache.org> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- .../scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala | 6 +++--- .../scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala | 6 +++--- .../scala/org/apache/spark/sql/jdbc/v2/MySQLNamespaceSuite.scala | 6 +++--- .../scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala | 3 ++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala index 2bde50d01eb..bc202b1b832 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala @@ -26,9 +26,9 @@ import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._ import org.apache.spark.tags.DockerTest /** - * To run this test suite for a specific version (e.g., mysql:5.7.36): + * To run this test suite for a specific version (e.g., mysql:8.0.31): * {{{ - * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:5.7.36 + * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:8.0.31 * ./build/sbt -Pdocker-integration-tests * "testOnly org.apache.spark.sql.jdbc.MySQLIntegrationSuite" * }}} @@ -36,7 +36,7 @@ import org.apache.spark.tags.DockerTest @DockerTest class MySQLIntegrationSuite extends DockerJDBCIntegrationSuite { override val db = new DatabaseOnDocker { - override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME", "mysql:5.7.36") + override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME", "mysql:8.0.31") override val env = Map( "MYSQL_ROOT_PASSWORD" -> "rootpass" ) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala index 6e76b74c7d8..072fdbb3f34 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLIntegrationSuite.scala @@ -29,9 +29,9 @@ import org.apache.spark.sql.types._ import org.apache.spark.tags.DockerTest /** - * To run this test suite for a specific version (e.g., mysql:5.7.36): + * To run this test suite for a specific version (e.g., mysql:8.0.31): * {{{ - * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:5.7.36 + * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:8.0.31 * ./build/sbt -Pdocker-integration-tests "testOnly *v2*MySQLIntegrationSuite" * }}} */ @@ -39,7 +39,7 @@ import org.apache.spark.tags.DockerTest class MySQLIntegrationSuite extends DockerJDBCIntegrationV2Suite with V2JDBCTest { override val catalogName: String = "mysql" override val db = new DatabaseOnDocker { - override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME", "mysql:5.7.36") + override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME", "mysql:8.0.31") override val env = Map( "MYSQL_ROOT_PASSWORD" -> "rootpass" ) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLNamespaceSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLNamespaceSuite.scala index d8dee61d70e..b73e2b8fd23 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLNamespaceSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/MySQLNamespaceSuite.scala @@ -28,16 +28,16 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.tags.DockerTest /** - * To run this test suite for a specific version (e.g., mysql:5.7.36): + * To run this test suite for a specific version (e.g., mysql:8.0.31): * {{{ - * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:5.7.36 + * ENABLE_DOCKER_INTEGRATION_TESTS=1 MYSQL_DOCKER_IMAGE_NAME=mysql:8.0.31 * ./build/sbt -Pdocker-integration-tests "testOnly *v2*MySQLNamespaceSuite" * }}} */ @DockerTest class MySQLNamespaceSuite extends DockerJDBCIntegrationSuite with V2JDBCNamespaceTest { override val db = new DatabaseOnDocker { - override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME", "mysql:5.7.36") + override val imageName = sys.env.getOrElse("MYSQL_DOCKER_IMAGE_NAME", "mysql:8.0.31") override val env = Map( "MYSQL_ROOT_PASSWORD" -> "rootpass" ) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala index 50e79e03a7a..b2500a2dbf2 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala @@ -62,7 +62,8 @@ private[v2] trait V2JDBCNamespaceTest extends SharedSparkSession with DockerInte Map.empty[String, String] } catalog.createNamespace(Array("foo"), commentMap.asJava) - assert(catalog.listNamespaces() === listNamespaces(Array("foo"))) + assert(catalog.listNamespaces().map(_.toSet).toSet === + listNamespaces(Array("foo")).map(_.toSet).toSet) assert(catalog.listNamespaces(Array("foo")) === Array()) assert(catalog.namespaceExists(Array("foo")) === true) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org