This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 34158d8 HIVE-25701: Declare JDBC drivers as runtime & optional
dependencies (Stamatis Zampetakis, reviewed by Zoltan Haindrich)
34158d8 is described below
commit 34158d8531725ab778a6c6112391370b44669bd8
Author: Stamatis Zampetakis <[email protected]>
AuthorDate: Mon Nov 15 14:13:43 2021 +0100
HIVE-25701: Declare JDBC drivers as runtime & optional dependencies
(Stamatis Zampetakis, reviewed by Zoltan Haindrich)
1. Use dependency management section for setting version, scope, and
optionality of a JDBC driver.
2. Gather declarations of JDBC drivers together.
3. Remove download-maven-plugin.
4. Add explicit dependencies to drivers when necessary.
5. Remove redundant itest.jdbc.jars property and update test
instructions
Declaring drivers as runtime & optional dependency has the following
advantages:
* Eliminates the risk to write code which needs JDBC driver classes in
order to compile and potentially violate AL2.
* Unifies the declaration of JDBC drivers making easier to add/remove
one if necessary.
* Removes the need to use download-maven-plugin and other similar
workarounds to avoid licensing problems.
* Simplifies the execution of tests using these drivers since now they
are added in the runtime classpath automatically by maven.
* Projects with dependencies depending on Hive will not inherit any JDBC
driver by default.
Closes #2790
---
Jenkinsfile | 2 +-
beeline/pom.xml | 2 -
itests/qtest-iceberg/pom.xml | 7 ---
itests/qtest/pom.xml | 34 ++----------
pom.xml | 52 +++++++++++++++---
.../hadoop/hive/ql/lockmgr/ITestDbTxnManager.java | 2 +-
standalone-metastore/DEV-README | 24 +++------
standalone-metastore/metastore-common/pom.xml | 1 -
standalone-metastore/metastore-server/pom.xml | 30 ++++-------
standalone-metastore/pom.xml | 62 +++++++++++++++-------
10 files changed, 112 insertions(+), 104 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index b3127c9..f7ccca4 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -272,7 +272,7 @@ time docker rm -f dev_$dbType || true
stage('verify') {
try {
sh """#!/bin/bash -e
-mvn verify -DskipITests=false -Dit.test=ITest${dbType.capitalize()}
-Dtest=nosuch -pl standalone-metastore/metastore-server
-Dmaven.test.failure.ignore -B -Ditest.jdbc.jars=`find /apps/lib/ -type f |
paste -s -d:`
+mvn verify -DskipITests=false -Dit.test=ITest${dbType.capitalize()}
-Dtest=nosuch -pl standalone-metastore/metastore-server
-Dmaven.test.failure.ignore -B
"""
} finally {
junit '**/TEST-*.xml'
diff --git a/beeline/pom.xml b/beeline/pom.xml
index 368d865..5617b4e 100644
--- a/beeline/pom.xml
+++ b/beeline/pom.xml
@@ -193,8 +193,6 @@
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
- <version>${postgres.version}</version>
- <scope>test</scope>
</dependency>
</dependencies>
diff --git a/itests/qtest-iceberg/pom.xml b/itests/qtest-iceberg/pom.xml
index e70aab4..839c714 100644
--- a/itests/qtest-iceberg/pom.xml
+++ b/itests/qtest-iceberg/pom.xml
@@ -382,18 +382,12 @@
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
- <!-- Note, this is LGPL. But we're only using it in a test and
not changing it, so I
- believe we are fine. -->
<groupId>org.mariadb.jdbc</groupId>
<artifactId>mariadb-java-client</artifactId>
- <version>${mariadb.version}</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
- <version>${postgres.version}</version>
- <scope>test</scope>
</dependency>
</dependencies>
@@ -446,7 +440,6 @@
<additionalClasspathElements>
<additionalClasspathElement>${test.conf.dir}</additionalClasspathElement>
<additionalClasspathElement>${basedir}/${hive.path.to.root}/conf</additionalClasspathElement>
-
<additionalClasspathElement>${itest.jdbc.jars}</additionalClasspathElement>
</additionalClasspathElements>
</configuration>
</plugin>
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index f772015..b86fffa 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -404,31 +404,26 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
+ <!-- JDBC drivers -->
+ <dependency>
+ <groupId>com.microsoft.sqlserver</groupId>
+ <artifactId>mssql-jdbc</artifactId>
+ </dependency>
<dependency>
- <!-- Note, this is LGPL. But we're only using it in a test and not
changing it, so I
- believe we are fine. -->
<groupId>org.mariadb.jdbc</groupId>
<artifactId>mariadb-java-client</artifactId>
- <version>${mariadb.version}</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
- <version>${postgres.version}</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
- <version>${mysql.version}</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc8</artifactId>
- <version>${oracle.version}</version>
- <scope>test</scope>
</dependency>
</dependencies>
<profiles>
@@ -490,23 +485,6 @@
<build>
<plugins>
<plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
- <version>1.3.0</version>
- <executions>
- <execution>
- <phase>generate-sources</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
-
<url>http://downloads.mariadb.com/Connectors/java/connector-java-2.2.0/mariadb-java-client-2.2.0.jar</url>
- <outputFileName>mariadb-java-client-2.2.0.jar</outputFileName>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>properties-maven-plugin</artifactId>
<version>1.0-alpha-2</version>
@@ -604,8 +582,6 @@
<additionalClasspathElements>
<additionalClasspathElement>${test.conf.dir}</additionalClasspathElement>
<additionalClasspathElement>${basedir}/${hive.path.to.root}/conf</additionalClasspathElement>
-
<additionalClasspathElement>${itest.jdbc.jars}</additionalClasspathElement>
-
<additionalClasspathElement>${project.build.directory}/mariadb-java-client-2.2.0.jar</additionalClasspathElement>
</additionalClasspathElements>
</configuration>
</plugin>
diff --git a/pom.xml b/pom.xml
index cd654d0..c5542e4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -80,8 +80,6 @@
<!-- Test Properties -->
<test.extra.path></test.extra.path>
-
<itest.jdbc.jars>set-this-to-colon-separated-full-path-list-of-jars-to-run-integration-tests
- </itest.jdbc.jars>
<!--suppress UnresolvedMavenProperty -->
<test.hive.hadoop.classpath>${maven.test.classpath}</test.hive.hadoop.classpath>
<test.log4j.scheme>file://</test.log4j.scheme>
@@ -182,6 +180,7 @@
<libthrift.version>0.14.1</libthrift.version>
<log4j2.version>2.13.2</log4j2.version>
<mariadb.version>2.5.0</mariadb.version>
+ <mssql.version>6.2.1.jre8</mssql.version>
<mysql.version>8.0.27</mysql.version>
<postgres.version>42.2.14</postgres.version>
<oracle.version>21.3.0.0</oracle.version>
@@ -586,11 +585,6 @@
</exclusions>
</dependency>
<dependency>
- <groupId>org.apache.derby</groupId>
- <artifactId>derby</artifactId>
- <version>${derby.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${httpcomponents.client.version}</version>
@@ -1274,6 +1268,49 @@
<groupId>org.apache.tez</groupId>
<artifactId>tez-common</artifactId>
<version>${tez.version}</version>
+ </dependency>
+ <!-- JDBC drivers -->
+ <dependency>
+ <groupId>com.microsoft.sqlserver</groupId>
+ <artifactId>mssql-jdbc</artifactId>
+ <version>${mssql.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>com.oracle.database.jdbc</groupId>
+ <artifactId>ojdbc8</artifactId>
+ <version>${oracle.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ <version>${derby.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.mariadb.jdbc</groupId>
+ <artifactId>mariadb-java-client</artifactId>
+ <version>${mariadb.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>${postgres.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
</dependency>
</dependencies>
</dependencyManagement>
@@ -1564,7 +1601,6 @@
<additionalClasspathElements>
<additionalClasspathElement>${test.conf.dir}</additionalClasspathElement>
<additionalClasspathElement>${basedir}/${hive.path.to.root}/conf</additionalClasspathElement>
-
<additionalClasspathElement>${itest.jdbc.jars}</additionalClasspathElement>
</additionalClasspathElements>
<environmentVariables>
<TZ>US/Pacific</TZ>
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/ITestDbTxnManager.java
b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/ITestDbTxnManager.java
index c19f222..e6806bd 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/ITestDbTxnManager.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/ITestDbTxnManager.java
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
/**
* Test class to run DbTxnManager tests against different dbms types.
- * Example: mvn test -Dtest=ITestDbTxnManager -Dtest.metastore.db=postgres
-Ditest.jdbc.jars=yourPathtoJdbcDriver
+ * Example: mvn test -Dtest=ITestDbTxnManager -Dtest.metastore.db=postgres
*/
public class ITestDbTxnManager extends TestDbTxnManager2 {
diff --git a/standalone-metastore/DEV-README b/standalone-metastore/DEV-README
index 69a5474..f0f5cac 100644
--- a/standalone-metastore/DEV-README
+++ b/standalone-metastore/DEV-README
@@ -23,25 +23,22 @@ quick test can be done for the unit tests and more in depth
testing as part
of the checkin tests.
--------------------------------------------------------------------------------
-Testing metastore installation and upgrade against databases beyond Derby:
There are integration tests for testing installation and upgrade of the
-metastore on MySQL (actually MariaDB is used), Oracle, Postgres, and SQLServer.
-These tests are not run by default because they take several minutes each and
-they require the developer to download the JDBC driver for Oracle.
-They are run in the integration-test phase.
+metastore on Derby, MySQL (actually MariaDB is used), Oracle, Postgres, and
SQLServer.
Each ITest runs two tests, one that installs the latest version of the
database and one that installs the latest version minus one and then upgrades
the database.
-To run one of the tests you will need to explicitly turn on integration
testing,
-in the Oracle case specify the location of the JDBC driver, and optionally
-specify which test you want to run. You'll need to download and start docker.
-Make sure docker's memory is set to more than 3.5GB. To run all of the tests
do:
+To run the tests you will need to explicitly turn on integration testing by
+setting skipITests variable to false. The tests rely on Docker so the latter
+needs to be installed and configured properly (e.g., memory more than 3.5GB).
-mvn verify -Ditest.jdbc.jars=_connector_jar_path -DskipITests=false
-Dtest=nosuch
+Run all tests:
-To run just one test, do
+mvn verify -DskipITests=false -Dtest=nosuch
+
+Run a single test:
mvn verify -DskipITests=false -Dit.test=ITestMysql -Dtest=nosuch
@@ -54,11 +51,6 @@ Supported databases for testing:
By adding -Dverbose.schematool the Schema Tool output becomes more detailed.
-Regarding connector jars:
- - You can download the Oracle driver at
- http://www.oracle.com/technetwork/database/features/jdbc/index-091264.html
- You should download Oracle 11g Release 1, ojdbc6.jar
-
Logs for tests are located under
standalone-metastore/metastore-common/target/failsafe-reports
If you wish to use one of these containers to run your own tests against a
diff --git a/standalone-metastore/metastore-common/pom.xml
b/standalone-metastore/metastore-common/pom.xml
index 4af0495..626e9c9 100644
--- a/standalone-metastore/metastore-common/pom.xml
+++ b/standalone-metastore/metastore-common/pom.xml
@@ -554,7 +554,6 @@
</systemPropertyVariables>
<additionalClasspathElements>
<additionalClasspathElement>${log4j.conf.dir}</additionalClasspathElement>
-
<additionalClasspathElement>${itest.jdbc.jars}</additionalClasspathElement>
</additionalClasspathElements>
<skipITs>${skipITests}</skipITs> <!-- set this to false to run these
tests -->
</configuration>
diff --git a/standalone-metastore/metastore-server/pom.xml
b/standalone-metastore/metastore-server/pom.xml
index 0ac64eb..4b17c2b 100644
--- a/standalone-metastore/metastore-server/pom.xml
+++ b/standalone-metastore/metastore-server/pom.xml
@@ -262,11 +262,19 @@
<groupId>com.cronutils</groupId>
<artifactId>cron-utils</artifactId>
</dependency>
+ <!-- JDBC drivers -->
+ <dependency>
+ <groupId>com.oracle.database.jdbc</groupId>
+ <artifactId>ojdbc8</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ </dependency>
<!-- test scope dependencies -->
<dependency>
<groupId>com.microsoft.sqlserver</groupId>
<artifactId>mssql-jdbc</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -300,7 +308,6 @@
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
@@ -453,23 +460,6 @@
<plugins>
<!-- plugins are always listed in sorted order by groupId, artifactId -->
<plugin>
- <groupId>com.googlecode.maven-download-plugin</groupId>
- <artifactId>download-maven-plugin</artifactId>
- <version>1.3.0</version>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>wget</goal>
- </goals>
- <configuration>
-
<url>http://downloads.mariadb.com/Connectors/java/connector-java-2.2.0/mariadb-java-client-2.2.0.jar</url>
- <outputFileName>mariadb-java-client-2.2.0.jar</outputFileName>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
@@ -612,8 +602,6 @@
</systemPropertyVariables>
<additionalClasspathElements>
<additionalClasspathElement>${log4j.conf.dir}</additionalClasspathElement>
-
<additionalClasspathElement>${itest.jdbc.jars}</additionalClasspathElement>
-
<additionalClasspathElement>${project.build.directory}/mariadb-java-client-2.2.0.jar</additionalClasspathElement>
</additionalClasspathElements>
<skipITs>${skipITests}</skipITs> <!-- set this to false to run these
tests -->
</configuration>
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index d7eaf6d..9b3d3a3 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -55,8 +55,6 @@
<test.warehouse.scheme>file://</test.warehouse.scheme>
<test.forkcount>1</test.forkcount>
<skipITests>true</skipITests>
-
<itest.jdbc.jars>set-this-to-colon-separated-full-path-list-of-jars-to-run-integration-tests
- </itest.jdbc.jars>
<test.groups>org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest</test.groups>
<!-- Plugin versions -->
@@ -75,7 +73,11 @@
<datanucleus-jdo.version>3.2.0-release</datanucleus-jdo.version>
<datanucleus-rdbms.version>5.2.4</datanucleus-rdbms.version>
<derby.version>10.14.1.0</derby.version>
+ <mariadb.version>2.5.0</mariadb.version>
+ <mssql.version>6.2.1.jre8</mssql.version>
+ <mysql.version>8.0.27</mysql.version>
<postgres.version>42.2.14</postgres.version>
+ <oracle.version>21.3.0.0</oracle.version>
<dropwizard-metrics-hadoop-metrics2-reporter.version>0.1.2
</dropwizard-metrics-hadoop-metrics2-reporter.version>
<dropwizard.version>3.1.0</dropwizard.version>
@@ -187,11 +189,6 @@
<version>${commons-lang3.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.derby</groupId>
- <artifactId>derby</artifactId>
- <version>${derby.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
@@ -331,15 +328,50 @@
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
-
- <!-- test scope dependencies -->
-
+ <!-- JDBC drivers -->
<dependency>
<groupId>com.microsoft.sqlserver</groupId>
<artifactId>mssql-jdbc</artifactId>
- <version>6.2.1.jre8</version>
- <scope>test</scope>
+ <version>${mssql.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>com.oracle.database.jdbc</groupId>
+ <artifactId>ojdbc8</artifactId>
+ <version>${oracle.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ <version>${derby.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.mariadb.jdbc</groupId>
+ <artifactId>mariadb-java-client</artifactId>
+ <version>${mariadb.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>${postgres.version}</version>
+ <scope>runtime</scope>
+ <optional>true</optional>
</dependency>
+ <!-- test scope dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
@@ -377,12 +409,6 @@
<version>${mockito-core.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.postgresql</groupId>
- <artifactId>postgresql</artifactId>
- <version>${postgres.version}</version>
- <scope>test</scope>
- </dependency>
<!-- https://mvnrepository.com/artifact/org.hamcrest/hamcrest-all -->
<dependency>
<groupId>org.hamcrest</groupId>
