This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 2dbfae8 [SPARK-32049][SQL][TESTS] Upgrade Oracle JDBC Driver 8
2dbfae8 is described below
commit 2dbfae8775e00da521f59c6328428ae541396802
Author: Gabor Somogyi <[email protected]>
AuthorDate: Tue Jun 23 03:58:40 2020 -0700
[SPARK-32049][SQL][TESTS] Upgrade Oracle JDBC Driver 8
### What changes were proposed in this pull request?
`OracleIntegrationSuite` is not using the latest oracle JDBC driver. In
this PR I've upgraded the driver to the latest which supports JDK8, JDK9, and
JDK11.
### Why are the changes needed?
Old JDBC driver.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing unit tests.
Existing integration tests (especially `OracleIntegrationSuite`)
Closes #28893 from gaborgsomogyi/SPARK-32049.
Authored-by: Gabor Somogyi <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
external/docker-integration-tests/pom.xml | 12 ++------
.../sql/jdbc/DockerJDBCIntegrationSuite.scala | 3 +-
.../spark/sql/jdbc/OracleIntegrationSuite.scala | 36 +++++++++++++---------
pom.xml | 6 ++++
4 files changed, 32 insertions(+), 25 deletions(-)
diff --git a/external/docker-integration-tests/pom.xml
b/external/docker-integration-tests/pom.xml
index 298e3d3..b240dd2 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -130,15 +130,9 @@
<artifactId>postgresql</artifactId>
<scope>test</scope>
</dependency>
- <!-- Oracle ojdbc jar, used for oracle integration suite for docker
testing.
- See https://github.com/apache/spark/pull/11306 for background on why we
need
- to use a an ojdbc jar for the testcase. The maven dependency here is
commented
- because currently the maven repository does not contain the ojdbc jar
mentioned.
- Once the jar is available in maven, this could be uncommented. -->
- <dependency>
- <groupId>com.oracle</groupId>
- <artifactId>ojdbc6</artifactId>
- <version>11.2.0.1.0</version>
+ <dependency>
+ <groupId>com.oracle.database.jdbc</groupId>
+ <artifactId>ojdbc8</artifactId>
<scope>test</scope>
</dependency>
diff --git
a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
index d15b366..6d1a22d 100644
---
a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
+++
b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
@@ -95,6 +95,7 @@ abstract class DockerJDBCIntegrationSuite extends
SharedSparkSession with Eventu
protected val dockerIp = DockerUtils.getDockerIp()
val db: DatabaseOnDocker
+ val connectionTimeout = timeout(2.minutes)
private var docker: DockerClient = _
protected var externalPort: Int = _
@@ -155,7 +156,7 @@ abstract class DockerJDBCIntegrationSuite extends
SharedSparkSession with Eventu
docker.startContainer(containerId)
jdbcUrl = db.getJdbcUrl(dockerIp, externalPort)
var conn: Connection = null
- eventually(timeout(2.minutes), interval(1.second)) {
+ eventually(connectionTimeout, interval(1.second)) {
conn = getConnection()
}
// Run any setup queries:
diff --git
a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
index 24c3adb..9c59023 100644
---
a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
+++
b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
@@ -21,6 +21,8 @@ import java.math.BigDecimal
import java.sql.{Connection, Date, Timestamp}
import java.util.{Properties, TimeZone}
+import org.scalatest.time.SpanSugar._
+
import org.apache.spark.sql.{Row, SaveMode}
import org.apache.spark.sql.execution.{RowDataSourceScanExec,
WholeStageCodegenExec}
import org.apache.spark.sql.execution.datasources.LogicalRelation
@@ -31,27 +33,27 @@ import org.apache.spark.sql.types._
import org.apache.spark.tags.DockerTest
/**
- * This patch was tested using the Oracle docker. Created this integration
suite for the same.
- * The ojdbc6-11.2.0.2.0.jar was to be downloaded from the maven repository.
Since there was
- * no jdbc jar available in the maven repository, the jar was downloaded from
oracle site
- * manually and installed in the local; thus tested. So, for SparkQA test case
run, the
- * ojdbc jar might be manually placed in the local maven
repository(com/oracle/ojdbc6/11.2.0.2.0)
- * while Spark QA test run.
- *
* The following would be the steps to test this
* 1. Build Oracle database in Docker, please refer below link about how to.
*
https://github.com/oracle/docker-images/blob/master/OracleDatabase/SingleInstance/README.md
* 2. export ORACLE_DOCKER_IMAGE_NAME=$ORACLE_DOCKER_IMAGE_NAME
* Pull oracle $ORACLE_DOCKER_IMAGE_NAME image - docker pull
$ORACLE_DOCKER_IMAGE_NAME
* 3. Start docker - sudo service docker start
- * 4. Download oracle 11g driver jar and put it in maven local repo:
- * (com/oracle/ojdbc6/11.2.0.2.0/ojdbc6-11.2.0.2.0.jar)
- * 5. The timeout and interval parameter to be increased from 60,1 to a high
value for oracle test
- * in DockerJDBCIntegrationSuite.scala (Locally tested with 200,200 and
executed successfully).
- * 6. Run spark test - ./build/sbt "test-only
org.apache.spark.sql.jdbc.OracleIntegrationSuite"
+ * 4. Run spark test - ./build/sbt -Pdocker-integration-tests
+ * "test-only org.apache.spark.sql.jdbc.OracleIntegrationSuite"
+ *
+ * An actual sequence of commands to run the test is as follows
*
- * All tests in this suite are ignored because of the dependency with the
oracle jar from maven
- * repository.
+ * $ git clone https://github.com/oracle/docker-images.git
+ * // Head SHA: 3e352a22618070595f823977a0fd1a3a8071a83c
+ * $ cd docker-images/OracleDatabase/SingleInstance/dockerfiles
+ * $ ./buildDockerImage.sh -v 18.4.0 -x
+ * $ export ORACLE_DOCKER_IMAGE_NAME=oracle/database:18.4.0-xe
+ * $ cd $SPARK_HOME
+ * $ ./build/sbt -Pdocker-integration-tests
+ * "test-only org.apache.spark.sql.jdbc.OracleIntegrationSuite"
+ *
+ * It has been validated with 18.4.0 Express Edition.
*/
@DockerTest
class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with
SharedSparkSession {
@@ -60,7 +62,7 @@ class OracleIntegrationSuite extends
DockerJDBCIntegrationSuite with SharedSpark
override val db = new DatabaseOnDocker {
override val imageName = sys.env("ORACLE_DOCKER_IMAGE_NAME")
override val env = Map(
- "ORACLE_ROOT_PASSWORD" -> "oracle"
+ "ORACLE_PWD" -> "oracle"
)
override val usesIpc = false
override val jdbcPort: Int = 1521
@@ -68,7 +70,11 @@ class OracleIntegrationSuite extends
DockerJDBCIntegrationSuite with SharedSpark
s"jdbc:oracle:thin:system/oracle@//$ip:$port/xe"
}
+ override val connectionTimeout = timeout(7.minutes)
+
override def dataPreparation(conn: Connection): Unit = {
+ // In 18.4.0 Express Edition auto commit is enabled by default.
+ conn.setAutoCommit(false)
conn.prepareStatement("CREATE TABLE datetime (id NUMBER(10), d DATE, t
TIMESTAMP)")
.executeUpdate()
conn.prepareStatement(
diff --git a/pom.xml b/pom.xml
index 7b68465..82c12ae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -985,6 +985,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>com.oracle.database.jdbc</groupId>
+ <artifactId>ojdbc8</artifactId>
+ <version>19.6.0.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>${curator.version}</version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
