This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d35de01 HIVE-25594: Setup JDBC databases in tests via qt:database
option (Stamatis Zampetakis, reviewed by Zoltan Haindrich, Alessandro Solimando)
d35de01 is described below
commit d35de014dd49fdcfe0aacb68e6c587beff6d1dea
Author: Stamatis Zampetakis <[email protected]>
AuthorDate: Wed Oct 6 12:18:18 2021 +0200
HIVE-25594: Setup JDBC databases in tests via qt:database option (Stamatis
Zampetakis, reviewed by Zoltan Haindrich, Alessandro Solimando)
Main feature changes:
1. Introduce QTestDatabaseHandler class for handling completely setup,
init and cleanup of dockerized DBMS by using qt:database option.
2. Drop TestMiniLlapExtDBCliDriver, with associated config, properties
etc, and adapt tests to use the newly introduced qt:database option
Code readability/extensibility changes:
3. Refactor AbstractExternalDB and subclasses: (i) turn setters to
getters; (ii) inline methods; (iii) reduce visibility when possibse
(iv) drop redundant comments; (v) drop unnecessary logging and
try/catch blocks; (vi) improve log messages, exceptions, and javadoc;
(vii) always include SQLLine standard output/error in the logs; (viii)
drop catch & print, log & throw patterns; (ix) use RuntimeException
instead of checked IOException; (x) replace hard-coded string with
class reference in logger; (xi) drop dependency to lang3.StringUtils
Minor bug fixes & improvements:
4. Ensure a root user is available when each database starts and rename
methods in AbstractExternalDB accordingly.
5. Do not create additional database users; it is not necessary.
6. Use correct driver and docker image for MySQL database.
7. Support additional databases (MariaDB, Oracle, MSSQLServer) for
tests.
Closes #2742
---
data/scripts/derby/q_test_extDB_cleanup.derby.sql | 3 -
data/scripts/derby/q_test_extDB_init.derby.sql | 27 ---
data/scripts/mysql/q_test_extDB_cleanup.mysql.sql | 3 -
.../postgres/q_test_extDB_cleanup.postgres.sql | 3 -
.../postgres/q_test_extDB_init.postgres.sql | 23 --
data/scripts/q_test_author_table.sql | 8 +
data/scripts/q_test_book_table.sql | 10 +
data/scripts/q_test_city_table.sql | 22 ++
...ql.sql => q_test_country_state_city_tables.sql} | 0
data/scripts/q_test_country_table.sql | 12 ++
data/scripts/q_test_state_table.sql | 18 ++
itests/qtest/pom.xml | 12 ++
.../hive/cli/TestMiniLlapExtDBCliDriver.java | 65 ------
.../test/resources/testconfiguration.properties | 5 -
.../hadoop/hive/cli/control/AbstractCliConfig.java | 6 -
.../apache/hadoop/hive/cli/control/CliConfigs.java | 24 ---
.../hadoop/hive/cli/control/CoreCliDriver.java | 3 -
.../org/apache/hadoop/hive/ql/QTestArguments.java | 17 --
.../org/apache/hadoop/hive/ql/QTestExternalDB.java | 57 -----
.../java/org/apache/hadoop/hive/ql/QTestUtil.java | 69 +-----
.../hive/ql/externalDB/AbstractExternalDB.java | 210 ++++++------------
.../hadoop/hive/ql/externalDB/MSSQLServer.java | 57 +++++
.../apache/hadoop/hive/ql/externalDB/MariaDB.java | 54 +++++
.../hadoop/hive/ql/externalDB/MySQLExternalDB.java | 29 +--
.../apache/hadoop/hive/ql/externalDB/Oracle.java | 56 +++++
.../hive/ql/externalDB/PostgresExternalDB.java | 20 +-
.../hive/ql/qoption/QTestDatabaseHandler.java | 135 ++++++++++++
.../ql/scheduled/QTestScheduledQueryCleaner.java | 7 +-
pom.xml | 2 +
.../queries/clientpositive/dataconnector_mysql.q | 3 +-
.../test/queries/clientpositive/qt_database_all.q | 86 ++++++++
.../queries/clientpositive/qt_database_mariadb.q | 17 ++
.../queries/clientpositive/qt_database_mssql.q | 16 ++
.../queries/clientpositive/qt_database_mysql.q | 16 ++
.../queries/clientpositive/qt_database_oracle.q | 17 ++
.../queries/clientpositive/qt_database_postgres.q | 16 ++
.../clientpositive/llap/dataconnector_mysql.q.out | 4 +-
.../clientpositive/llap/qt_database_all.q.out | 235 +++++++++++++++++++++
.../clientpositive/llap/qt_database_mariadb.q.out | 45 ++++
.../clientpositive/llap/qt_database_mssql.q.out | 45 ++++
.../clientpositive/llap/qt_database_mysql.q.out | 45 ++++
.../clientpositive/llap/qt_database_oracle.q.out | 45 ++++
.../clientpositive/llap/qt_database_postgres.q.out | 45 ++++
43 files changed, 1117 insertions(+), 475 deletions(-)
diff --git a/data/scripts/derby/q_test_extDB_cleanup.derby.sql
b/data/scripts/derby/q_test_extDB_cleanup.derby.sql
deleted file mode 100644
index e62f1f0..0000000
--- a/data/scripts/derby/q_test_extDB_cleanup.derby.sql
+++ /dev/null
@@ -1,3 +0,0 @@
-drop table country;
-drop table stats;
-drop table city;
diff --git a/data/scripts/derby/q_test_extDB_init.derby.sql
b/data/scripts/derby/q_test_extDB_init.derby.sql
deleted file mode 100644
index b2a90d7..0000000
--- a/data/scripts/derby/q_test_extDB_init.derby.sql
+++ /dev/null
@@ -1,27 +0,0 @@
--- reference:
https://medium.com/swlh/getting-started-with-apache-derby-9f6ca8dea873
-
-create table country (name char(30) not null unique);
-create table state (name char(30) not null unique,
- country_code int not null);
-create table city (name char(30) not null unique,
- state_code int not null);
-
-insert into country (name) values ('India');
-insert into country (name) values ('Russia');
-insert into country (name) values ('USA');
-
-insert into state (name,country_code) values ('Maharashtra',1);
-insert into state (name,country_code) values ('Madhya Pradesh',1);
-insert into state (name,country_code) values ('Moscow',3);
-insert into state (name,country_code) values ('Something',4);
-insert into state (name,country_code) values ('Florida',4);
-insert into state (name,country_code) values ('Texas',4);
-
-insert into city (name,state_code) values('Mumbai',1);
-insert into city (name,state_code) values('Pune',1);
-insert into city (name,state_code) values('Bhopal',2);
-insert into city (name,state_code) values('Indore',2);
-insert into city (name,state_code) values('Klin',3);
-insert into city (name,state_code) values('Los Angeles',5);
-insert into city (name,state_code) values('Plant City',5);
-insert into city (name,state_code) values('Arlington',6);
diff --git a/data/scripts/mysql/q_test_extDB_cleanup.mysql.sql
b/data/scripts/mysql/q_test_extDB_cleanup.mysql.sql
deleted file mode 100644
index abe1cae..0000000
--- a/data/scripts/mysql/q_test_extDB_cleanup.mysql.sql
+++ /dev/null
@@ -1,3 +0,0 @@
-drop table if exists country;
-drop table if exists stats;
-drop table if exists city;
diff --git a/data/scripts/postgres/q_test_extDB_cleanup.postgres.sql
b/data/scripts/postgres/q_test_extDB_cleanup.postgres.sql
deleted file mode 100644
index abe1cae..0000000
--- a/data/scripts/postgres/q_test_extDB_cleanup.postgres.sql
+++ /dev/null
@@ -1,3 +0,0 @@
-drop table if exists country;
-drop table if exists stats;
-drop table if exists city;
diff --git a/data/scripts/postgres/q_test_extDB_init.postgres.sql
b/data/scripts/postgres/q_test_extDB_init.postgres.sql
deleted file mode 100644
index a9cb486..0000000
--- a/data/scripts/postgres/q_test_extDB_init.postgres.sql
+++ /dev/null
@@ -1,23 +0,0 @@
-create table if not exists country (name varchar(80) not null);
-create table if not exists state (name varchar(80) not null, country_code int
not null);
-create table if not exists city (name varchar(80) not null, state_code int not
null);
-
-insert into country (name) values ('India');
-insert into country (name) values ('Russia');
-insert into country (name) values ('USA');
-
-insert into state (name,country_code) values ('Maharashtra',1);
-insert into state (name,country_code) values ('Madhya Pradesh',1);
-insert into state (name,country_code) values ('Moscow',3);
-insert into state (name,country_code) values ('Something',4);
-insert into state (name,country_code) values ('Florida',4);
-insert into state (name,country_code) values ('Texas',4);
-
-insert into city (name,state_code) values('Mumbai',1);
-insert into city (name,state_code) values('Pune',1);
-insert into city (name,state_code) values('Bhopal',2);
-insert into city (name,state_code) values('Indore',2);
-insert into city (name,state_code) values('Klin',3);
-insert into city (name,state_code) values('Los Angeles',5);
-insert into city (name,state_code) values('Plant City',5);
-insert into city (name,state_code) values('Arlington',6);
diff --git a/data/scripts/q_test_author_table.sql
b/data/scripts/q_test_author_table.sql
new file mode 100644
index 0000000..8f5e0d0
--- /dev/null
+++ b/data/scripts/q_test_author_table.sql
@@ -0,0 +1,8 @@
+create table author
+(
+ id int,
+ fname varchar(20),
+ lname varchar(20)
+);
+insert into author values (1, 'Victor', 'Hugo');
+insert into author values (2, 'Alexandre', 'Dumas');
diff --git a/data/scripts/q_test_book_table.sql
b/data/scripts/q_test_book_table.sql
new file mode 100644
index 0000000..37686e1
--- /dev/null
+++ b/data/scripts/q_test_book_table.sql
@@ -0,0 +1,10 @@
+create table book
+(
+ id int,
+ title varchar(100),
+ author int
+);
+insert into book
+values (1, 'Les Miserables', 1);
+insert into book
+values (2, 'The Count Of Monte Cristo', 2);
diff --git a/data/scripts/q_test_city_table.sql
b/data/scripts/q_test_city_table.sql
new file mode 100644
index 0000000..56a76a5
--- /dev/null
+++ b/data/scripts/q_test_city_table.sql
@@ -0,0 +1,22 @@
+create table if not exists city
+(
+ name varchar(255) not null,
+ state int not null
+);
+
+insert into city (name, state)
+values ('Mumbai', 1);
+insert into city (name, state)
+values ('Pune', 1);
+insert into city (name, state)
+values ('Bhopal', 2);
+insert into city (name, state)
+values ('Indore', 2);
+insert into city (name, state)
+values ('Klin', 3);
+insert into city (name, state)
+values ('Los Angeles', 5);
+insert into city (name, state)
+values ('Plant City', 5);
+insert into city (name, state)
+values ('Arlington', 6);
diff --git a/data/scripts/mysql/q_test_extDB_init.mysql.sql
b/data/scripts/q_test_country_state_city_tables.sql
similarity index 100%
rename from data/scripts/mysql/q_test_extDB_init.mysql.sql
rename to data/scripts/q_test_country_state_city_tables.sql
diff --git a/data/scripts/q_test_country_table.sql
b/data/scripts/q_test_country_table.sql
new file mode 100644
index 0000000..ce8010b
--- /dev/null
+++ b/data/scripts/q_test_country_table.sql
@@ -0,0 +1,12 @@
+CREATE TABLE country
+(
+ id int,
+ name varchar(20)
+);
+
+insert into country
+values (1, 'India');
+insert into country
+values (2, 'Russia');
+insert into country
+values (3, 'USA');
diff --git a/data/scripts/q_test_state_table.sql
b/data/scripts/q_test_state_table.sql
new file mode 100644
index 0000000..3f3edba
--- /dev/null
+++ b/data/scripts/q_test_state_table.sql
@@ -0,0 +1,18 @@
+create table state
+(
+ name varchar(255) not null,
+ country int not null
+);
+
+insert into state (name, country)
+values ('Maharashtra', 1);
+insert into state (name, country)
+values ('Madhya Pradesh', 1);
+insert into state (name, country)
+values ('Moscow', 3);
+insert into state (name, country)
+values ('Something', 4);
+insert into state (name, country)
+values ('Florida', 4);
+insert into state (name, country)
+values ('Texas', 4);
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 53326dd..f772015 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -418,6 +418,18 @@
<version>${postgres.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.oracle.database.jdbc</groupId>
+ <artifactId>ojdbc8</artifactId>
+ <version>${oracle.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
<profile>
diff --git
a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapExtDBCliDriver.java
b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapExtDBCliDriver.java
deleted file mode 100644
index e1c804c..0000000
---
a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniLlapExtDBCliDriver.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.cli;
-
-import java.io.File;
-import java.util.List;
-
-import org.apache.hadoop.hive.cli.control.CliAdapter;
-import org.apache.hadoop.hive.cli.control.CliConfigs;
-import org.apache.hadoop.hive.cli.control.SplitSupport;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-@RunWith(Parameterized.class)
-public class TestMiniLlapExtDBCliDriver {
-
- static CliAdapter adapter = new
CliConfigs.MiniLlapExtDBCliConfig().getCliAdapter();
-
- private static int N_SPLITS = 3;
-
- @Parameters(name = "{0}")
- public static List<Object[]> getParameters() throws Exception {
- return SplitSupport.process(adapter.getParameters(),
TestMiniLlapExtDBCliDriver.class, N_SPLITS);
- }
-
- @ClassRule
- public static TestRule cliClassRule = adapter.buildClassRule();
-
- @Rule
- public TestRule cliTestRule = adapter.buildTestRule();
-
- private String name;
- private File qfile;
-
- public TestMiniLlapExtDBCliDriver(String name, File qfile) {
- this.name = name;
- this.qfile = qfile;
- }
-
- @Test
- public void testCliDriver() throws Exception {
- adapter.runTest(name, qfile);
- }
-
-}
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 71b6fed..638af07 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -1251,11 +1251,6 @@ erasurecoding.only.query.files=\
erasure_explain.q,\
erasure_simple.q
-# tests that requires external database connection
-externalDB.llap.query.files=\
- dataconnector.q,\
- dataconnector_mysql.q
-
iceberg.llap.query.files=\
llap_iceberg_read.q,\
vectorized_iceberg_read.q
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
index e21e40b..cc31ce9 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
@@ -36,7 +36,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.ql.QTestSystemProperties;
import org.apache.hadoop.hive.ql.QTestMiniClusters.FsType;
import org.apache.hadoop.hive.ql.QTestMiniClusters.MiniClusterType;
-import org.apache.hadoop.hive.ql.QTestExternalDB;
import org.apache.hive.testutils.HiveTestEnvSetup;
import com.google.common.base.Splitter;
@@ -61,7 +60,6 @@ public abstract class AbstractCliConfig {
// these should have viable defaults
private String cleanupScript;
private String initScript;
- private Set<QTestExternalDB> externalDBs = new LinkedHashSet<>();
private String hiveConfDir;
private MiniClusterType clusterType;
private FsType fsType;
@@ -344,10 +342,6 @@ public abstract class AbstractCliConfig {
this.initScript = initScript;
}
}
- public Set<QTestExternalDB> getExternalDBs() { return externalDBs; }
-
- protected void addExternalDB (QTestExternalDB externalDB) {
this.externalDBs.add(externalDB); }
-
public String getHiveConfDir() {
return hiveConfDir;
}
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index 3dbcf28..b69465c 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -24,7 +24,6 @@ import java.net.URL;
import org.apache.hadoop.hive.ql.QTestMiniClusters;
import org.apache.hadoop.hive.ql.QTestMiniClusters.MiniClusterType;
import org.apache.hadoop.hive.ql.parse.CoreParseNegative;
-import org.apache.hadoop.hive.ql.QTestExternalDB;
public class CliConfigs {
@@ -226,7 +225,6 @@ public class CliConfigs {
excludesFrom(testConfigProps, "spark.only.query.files");
excludesFrom(testConfigProps, "localSpark.only.query.files");
excludesFrom(testConfigProps, "miniSparkOnYarn.only.query.files");
- excludesFrom(testConfigProps, "externalDB.llap.query.files");
setResultsDir("ql/src/test/results/clientpositive/llap");
setLogDir("itests/qtest/target/qfile-results/clientpositive");
@@ -242,28 +240,6 @@ public class CliConfigs {
}
}
- public static class MiniLlapExtDBCliConfig extends AbstractCliConfig {
-
- public MiniLlapExtDBCliConfig() {
- super(CoreCliDriver.class);
- try {
- setQueryDir("ql/src/test/queries/clientpositive");
-
- includesFrom(testConfigProps, "externalDB.llap.query.files");
-
- setResultsDir("ql/src/test/results/clientpositive/llap");
- setLogDir("itests/qtest/target/qfile-results/clientpositive");
-
- addExternalDB(QTestExternalDB.createDefaultExtDB("mysql"));
-
- setHiveConfDir("data/conf/llap");
- setClusterType(MiniClusterType.LLAP_LOCAL);
- } catch (Exception e) {
- throw new RuntimeException("can't construct cliconfig", e);
- }
- }
- }
-
public static class EncryptedHDFSCliConfig extends AbstractCliConfig {
public EncryptedHDFSCliConfig() {
super(CoreCliDriver.class);
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
index c868ce2..b1ee326 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
@@ -25,7 +25,6 @@ import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.ql.QTestExternalDB;
import org.apache.hadoop.hive.ql.QTestArguments;
import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
@@ -63,7 +62,6 @@ public class CoreCliDriver extends CliAdapter {
String hiveConfDir = cliConfig.getHiveConfDir();
String initScript = cliConfig.getInitScript();
String cleanupScript = cliConfig.getCleanupScript();
- Set<QTestExternalDB> externalDBs = cliConfig.getExternalDBs();
qt = new ElapsedTimeLoggingWrapper<QTestUtil>() {
@Override
@@ -76,7 +74,6 @@ public class CoreCliDriver extends CliAdapter {
.withConfDir(hiveConfDir)
.withInitScript(initScript)
.withCleanupScript(cleanupScript)
- .withExternalDBs(externalDBs)
.withLlapIo(true)
.withFsType(cliConfig.getFsType())
.build());
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestArguments.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestArguments.java
index 5c59b92..2941efa 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestArguments.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestArguments.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.hive.ql;
-import java.util.Set;
-import java.util.LinkedHashSet;
-import org.apache.hadoop.hive.ql.QTestExternalDB;
import org.apache.hadoop.hive.ql.QTestMiniClusters.FsType;
import org.apache.hadoop.hive.ql.QTestMiniClusters.QTestSetup;
@@ -37,7 +34,6 @@ public final class QTestArguments {
private String cleanupScript;
private boolean withLlapIo;
private FsType fsType;
- private Set<QTestExternalDB> externalDBs = new LinkedHashSet<>();
private QTestSetup qtestSetup;
private QTestArguments() {
@@ -83,10 +79,6 @@ public final class QTestArguments {
return initScript;
}
- private void setExternalDBs(Set<QTestExternalDB> externalDBs) {
this.externalDBs = externalDBs; }
-
- public Set<QTestExternalDB> getExternalDBs() { return this.externalDBs; }
-
private void setCleanupScript(String cleanupScript) {
this.cleanupScript = cleanupScript;
}
@@ -130,7 +122,6 @@ public final class QTestArguments {
private QTestMiniClusters.MiniClusterType clusterType;
private String initScript;
private String cleanupScript;
- private Set<QTestExternalDB> externalDBs = new LinkedHashSet<>();
private boolean withLlapIo;
private FsType fsType;
private QTestSetup qtestSetup;
@@ -172,11 +163,6 @@ public final class QTestArguments {
return this;
}
- public QTestArgumentsBuilder withExternalDBs(Set<QTestExternalDB>
externalDBs) {
- this.externalDBs = externalDBs;
- return this;
- }
-
public QTestArgumentsBuilder withLlapIo(boolean withLlapIo) {
this.withLlapIo = withLlapIo;
return this;
@@ -202,9 +188,6 @@ public final class QTestArguments {
testArguments.setCleanupScript(cleanupScript);
testArguments.setWithLlapIo(withLlapIo);
- testArguments.setExternalDBs(
- externalDBs != null ? externalDBs : new LinkedHashSet<>());
-
testArguments.setFsType(
fsType != null ? fsType : clusterType.getDefaultFsType());
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestExternalDB.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestExternalDB.java
deleted file mode 100644
index f50955b..0000000
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestExternalDB.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql;
-
-import java.io.File;
-
-/**
- * QTestExternalDB composite used as information holder for creating external
databases of different types.
- */
-public final class QTestExternalDB {
-
- private String externalDBType;
- private String externalDBInitScript;
- private String externalDBCleanupScript;
-
- public QTestExternalDB() { }
-
- public static QTestExternalDB createDefaultExtDB(String externalDBType) {
-
- QTestExternalDB externalDB = new QTestExternalDB();
-
- externalDB.setExternalDBType(externalDBType);
-
externalDB.setExternalDBInitScript(String.format("q_test_extDB_init.%s.sql",
externalDBType));
-
externalDB.setExternalDBCleanupScript(String.format("q_test_extDB_cleanup.%s.sql",
externalDBType));
-
- return externalDB;
- }
-
- public void setExternalDBType(String dbType) { this.externalDBType =
dbType; }
-
- public String getExternalDBType() { return externalDBType; }
-
- public void setExternalDBInitScript(String InitScript) {
this.externalDBInitScript = InitScript; }
-
- public String getExternalDBInitScript() { return externalDBInitScript; }
-
- public void setExternalDBCleanupScript(String cleanupScript) {
this.externalDBCleanupScript = cleanupScript; }
-
- public String getExternalDBCleanupScript() { return
externalDBCleanupScript; }
-
-}
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index a9e6b05..e6c1624 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -80,6 +80,7 @@ import
org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.processors.HiveCommand;
import org.apache.hadoop.hive.ql.qoption.QTestAuthorizerHandler;
import org.apache.hadoop.hive.ql.qoption.QTestDisabledHandler;
+import org.apache.hadoop.hive.ql.qoption.QTestDatabaseHandler;
import org.apache.hadoop.hive.ql.qoption.QTestOptionDispatcher;
import org.apache.hadoop.hive.ql.qoption.QTestReplaceHandler;
import org.apache.hadoop.hive.ql.qoption.QTestSysDbHandler;
@@ -88,7 +89,6 @@ import org.apache.hadoop.hive.ql.qoption.QTestTransactional;
import org.apache.hadoop.hive.ql.scheduled.QTestScheduledQueryCleaner;
import org.apache.hadoop.hive.ql.scheduled.QTestScheduledQueryServiceProvider;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.ql.externalDB.*;
import org.apache.hive.common.util.ProcessUtils;
import org.junit.Assert;
import org.slf4j.Logger;
@@ -130,7 +130,6 @@ public class QTestUtil {
protected QTestReplaceHandler replaceHandler;
private final String initScript;
private final String cleanupScript;
- private final Set<QTestExternalDB> externalDBs = new LinkedHashSet<>();
QTestOptionDispatcher dispatcher = new QTestOptionDispatcher();
private boolean isSessionStateStarted = false;
@@ -181,14 +180,13 @@ public class QTestUtil {
public QTestUtil(QTestArguments testArgs) throws Exception {
LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={},
confDir={},"
- + " initScript={}, cleanupScript={}, externalDBs={},
withLlapIo={}, fsType={}",
+ + " initScript={}, cleanupScript={}, withLlapIo={}, fsType={}",
testArgs.getOutDir(),
testArgs.getLogDir(),
testArgs.getClusterType(),
testArgs.getConfDir(),
testArgs.getInitScript(),
testArgs.getCleanupScript(),
- testArgs.getExternalDBs(),
testArgs.isWithLlapIo(),
testArgs.getFsType());
@@ -218,7 +216,8 @@ public class QTestUtil {
conf = new HiveConf(IDriver.class);
setMetaStoreProperties();
- this.miniClusters.setup(testArgs, conf, getScriptsDir(), logDir);
+ final String scriptsDir = getScriptsDir(conf);
+ this.miniClusters.setup(testArgs, conf, scriptsDir, logDir);
initConf();
@@ -235,21 +234,11 @@ public class QTestUtil {
dispatcher.register("timezone", new QTestTimezoneHandler());
dispatcher.register("authorizer", new QTestAuthorizerHandler());
dispatcher.register("disabled", new QTestDisabledHandler());
-
- String scriptsDir = getScriptsDir();
+ dispatcher.register("database", new QTestDatabaseHandler());
this.initScript = scriptsDir + File.separator + testArgs.getInitScript();
this.cleanupScript = scriptsDir + File.separator +
testArgs.getCleanupScript();
- QTestExternalDB newExternalDB;
- for (QTestExternalDB externalDB : testArgs.getExternalDBs()) {
- newExternalDB = new QTestExternalDB();
- newExternalDB.setExternalDBType(externalDB.getExternalDBType());
- newExternalDB.setExternalDBInitScript(scriptsDir + File.separator +
externalDB.getExternalDBType() + File.separator +
externalDB.getExternalDBInitScript());
- newExternalDB.setExternalDBCleanupScript(scriptsDir + File.separator +
externalDB.getExternalDBType() + File.separator +
externalDB.getExternalDBCleanupScript());
- this.externalDBs.add(newExternalDB);
- }
-
savedConf = new HiveConf(conf);
}
@@ -278,7 +267,7 @@ public class QTestUtil {
}
}
- private String getScriptsDir() {
+ public static String getScriptsDir(HiveConf conf) {
// Use the current directory if it is not specified
String scriptsDir = conf.get("test.data.scripts");
if (scriptsDir == null) {
@@ -493,14 +482,6 @@ public class QTestUtil {
}
conf.setBoolean("hive.test.shutdown.phase", true);
- // if has external database, clean up docker container
- // connect to externalDB if size is not zero
- if (this.externalDBs != null && this.externalDBs.size() != 0) {
- for (QTestExternalDB externalDB : this.externalDBs) {
- externalDBCleanupContainer(externalDB.getExternalDBType());
- }
- }
-
clearKeysCreatedInTests();
String metastoreDb = QTestSystemProperties.getMetaStoreDb();
@@ -528,18 +509,6 @@ public class QTestUtil {
FunctionRegistry.unregisterTemporaryUDF("test_error");
}
- private void externalDBCleanupContainer(String externalDBType) throws
IOException {
- // get externalDB initScript
- try {
- AbstractExternalDB abstractExternalDB =
AbstractExternalDB.initalizeExternalDB(externalDBType);
- abstractExternalDB.cleanupDockerContainer();
- LOG.info("cleanup externalDB docker container succeeed!");
- } catch (Exception e) {
- LOG.info("cleanup externalDB failed: " + e.getMessage());
- Assert.fail("Failed during cleanup externalDB docker container");
- }
- }
-
private void cleanupFromFile() throws IOException {
File cleanupFile = new File(cleanupScript);
if (cleanupFile.isFile()) {
@@ -562,13 +531,6 @@ public class QTestUtil {
startSessionState(qTestResultProcessor.canReuseSession());
}
- // connect to externalDB if size is not zero
- if (this.externalDBs != null && this.externalDBs.size() != 0) {
- for (QTestExternalDB externalDB : this.externalDBs) {
- externalDBLoadFromScript(externalDB.getExternalDBType(),
externalDB.getExternalDBInitScript());
- }
- }
-
getCliDriver().processLine("set test.data.dir=" + testFiles + ";");
conf.setBoolean("hive.test.init.phase", true);
@@ -578,25 +540,6 @@ public class QTestUtil {
conf.setBoolean("hive.test.init.phase", false);
}
- private void externalDBLoadFromScript(String externalDBType, String
externalDBScript) throws IOException {
- // get externalDB initScript
- File scriptFile = new File(externalDBScript);
- if (!scriptFile.isFile()) {
- Assert.fail("cannot getting externaldb scirpt file");
- LOG.info("No externalDB init script detected. Skipping");
- return;
- }
-
- try {
- AbstractExternalDB abstractExternalDB =
AbstractExternalDB.initalizeExternalDB(externalDBType);
- abstractExternalDB.launchDockerContainer();
- abstractExternalDB.execute(externalDBScript);
- LOG.info("initialize external databases succeed!");
- } catch (Exception e) {
- Assert.fail("Failed during initialize external database");
- }
- }
-
private void initFromScript() throws IOException {
File scriptFile = new File(this.initScript);
if (!scriptFile.isFile()) {
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java
index fd96776..ba0c7ad 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/AbstractExternalDB.java
@@ -17,49 +17,37 @@
*/
package org.apache.hadoop.hive.ql.externalDB;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import sqlline.SqlLine;
-import org.apache.commons.io.output.NullOutputStream;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
import java.io.BufferedReader;
-import java.io.InputStreamReader;
import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileReader;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PrintStream;
import java.io.IOException;
-import java.net.URI;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
-
-import org.apache.commons.lang3.StringUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
/**
- * abstractExternalDB is incharge of connect to and populate externaal
database for qtest
+ * The class is in charge of connecting and populating dockerized databases
for qtest.
+ *
+ * The database should have at least one root user (admin/superuser) able to
modify every aspect of the system. The user
+ * either exists by default when the database starts or must created right
after startup.
*/
public abstract class AbstractExternalDB {
- protected static final Logger LOG =
LoggerFactory.getLogger("AbstractExternalDB");
+ protected static final Logger LOG =
LoggerFactory.getLogger(AbstractExternalDB.class);
- protected static final String userName = "qtestuser";
- protected static final String password = "qtestpassword";
protected static final String dbName = "qtestDB";
- public String externalDBType = "mysql"; // default: mysql
- protected String url = "jdbc:mysql://localhost:3306/" + dbName; //
default: mysql
- protected String driver = "org.mariadb.jdbc.Driver"; // default: mysql
private static final int MAX_STARTUP_WAIT = 5 * 60 * 1000;
- public static class ProcessResults {
+ protected static class ProcessResults {
final String stdout;
final String stderr;
final int rc;
@@ -71,27 +59,8 @@ public abstract class AbstractExternalDB {
}
}
- public static AbstractExternalDB initalizeExternalDB(String
externalDBType) throws IOException {
- AbstractExternalDB abstractExternalDB;
- switch (externalDBType) {
- case "mysql":
- abstractExternalDB = new MySQLExternalDB();
- break;
- case "postgres":
- abstractExternalDB = new PostgresExternalDB();
- break;
- default:
- throw new IOException("unsupported external database type " +
externalDBType);
- }
- return abstractExternalDB;
- }
-
- public AbstractExternalDB(String externalDBType) {
- this.externalDBType = externalDBType;
- }
-
- protected String getDockerContainerName() {
- return String.format("qtestExternalDB-%s", externalDBType);
+ private final String getDockerContainerName() {
+ return String.format("qtestExternalDB-%s", getClass().getSimpleName());
}
private String[] buildRunCmd() {
@@ -107,27 +76,17 @@ public abstract class AbstractExternalDB {
}
private String[] buildRmCmd() {
- return buildArray(
- "docker",
- "rm",
- "-f",
- "-v",
- getDockerContainerName()
- );
+ return new String[] { "docker", "rm", "-f", "-v",
getDockerContainerName() };
}
private String[] buildLogCmd() {
- return buildArray(
- "docker",
- "logs",
- getDockerContainerName()
- );
+ return new String[] { "docker", "logs", getDockerContainerName() };
}
private ProcessResults runCmd(String[] cmd, long secondsToWait)
throws IOException, InterruptedException {
- LOG.info("Going to run: " + StringUtils.join(cmd, " "));
+ LOG.info("Going to run: " + String.join(" ", cmd));
Process proc = Runtime.getRuntime().exec(cmd);
if (!proc.waitFor(secondsToWait, TimeUnit.SECONDS)) {
throw new RuntimeException(
@@ -152,7 +111,7 @@ public abstract class AbstractExternalDB {
}
- public void launchDockerContainer() throws Exception { //runDockerContainer
+ public void launchDockerContainer() throws Exception {
runCmdAndPrintStreams(buildRmCmd(), 600);
if (runCmdAndPrintStreams(buildRunCmd(), 600) != 0) {
throw new RuntimeException("Unable to start docker container");
@@ -172,19 +131,14 @@ public abstract class AbstractExternalDB {
}
}
- public void cleanupDockerContainer() { // stopAndRmDockerContainer
- try {
- if (runCmdAndPrintStreams(buildRmCmd(), 600) != 0) {
- LOG.info("Unable to remove docker container");
- throw new RuntimeException("Unable to remove docker
container");
- }
- } catch (InterruptedException | IOException e) {
- e.printStackTrace();
+ public void cleanupDockerContainer() throws IOException,
InterruptedException {
+ if (runCmdAndPrintStreams(buildRmCmd(), 600) != 0) {
+ throw new RuntimeException("Unable to remove docker container");
}
}
- public final String getContainerHostAddress() {
+ protected final String getContainerHostAddress() {
String hostAddress = System.getenv("HIVE_TEST_DOCKER_HOST");
if (hostAddress != null) {
return hostAddress;
@@ -193,96 +147,62 @@ public abstract class AbstractExternalDB {
}
}
- public abstract void setJdbcUrl(String hostAddress);
-
- public abstract void setJdbcDriver();
-
- public abstract String getDockerImageName();
-
- public abstract String[] getDockerAdditionalArgs();
-
- public abstract boolean isContainerReady(ProcessResults pr);
+ /**
+ * Return the name of the root user.
+ *
+ * Override the method if the name of the root user must be different than
the default.
+ */
+ protected String getRootUser() {
+ return "qtestuser";
+ }
- protected String[] buildArray(String... strs) {
- return strs;
+ /**
+ * Return the password of the root user.
+ *
+ * Override the method if the password must be different than the default.
+ */
+ protected String getRootPassword() {
+ return "qtestpassword";
}
+
+ protected abstract String getJdbcUrl();
- public Connection getConnectionToExternalDB() throws SQLException,
ClassNotFoundException {
- try {
- LOG.info("external database connection URL:\t " + url);
- LOG.info("JDBC Driver :\t " + driver);
- LOG.info("external database connection User:\t " + userName);
- LOG.info("external database connection Password:\t " + password);
+ protected abstract String getJdbcDriver();
- // load required JDBC driver
- Class.forName(driver);
+ protected abstract String getDockerImageName();
- // Connect using the JDBC URL and user/password
- Connection conn = DriverManager.getConnection(url, userName,
password);
- return conn;
- } catch (SQLException e) {
- LOG.error("Failed to connect to external databse", e);
- throw new SQLException(e);
- } catch (ClassNotFoundException e) {
- LOG.error("Unable to find driver class", e);
- throw new ClassNotFoundException("Unable to find driver class");
- }
- }
+ protected abstract String[] getDockerAdditionalArgs();
- public void testConnectionToExternalDB() throws SQLException,
ClassNotFoundException {
- Connection conn = getConnectionToExternalDB();
- try {
- conn.close();
- } catch (SQLException e) {
- LOG.error("Failed to close external database connection", e);
- }
- }
+ protected abstract boolean isContainerReady(ProcessResults pr);
- protected String[] SQLLineCmdBuild(String sqlScriptFile) throws
IOException {
- return new String[] {"-u", url,
- "-d", driver,
- "-n", userName,
- "-p", password,
+ private String[] SQLLineCmdBuild(String sqlScriptFile) {
+ return new String[] {"-u", getJdbcUrl(),
+ "-d", getJdbcDriver(),
+ "-n", getRootUser(),
+ "-p", getRootPassword(),
"--isolation=TRANSACTION_READ_COMMITTED",
"-f", sqlScriptFile};
}
- protected void execSql(String sqlScriptFile) throws IOException {
- // run the script using SqlLine
- SqlLine sqlLine = new SqlLine();
- ByteArrayOutputStream outputForLog = null;
- OutputStream out;
- if (LOG.isDebugEnabled()) {
- out = outputForLog = new ByteArrayOutputStream();
- } else {
- out = new NullOutputStream();
+ public void execute(String script) throws IOException, SQLException,
ClassNotFoundException {
+ // Test we can connect to database
+ Class.forName(getJdbcDriver());
+ try (Connection ignored = DriverManager.getConnection(getJdbcUrl(),
getRootUser(), getRootPassword())) {
+ LOG.info("Successfully connected to {} with user {} and password
{}", getJdbcUrl(), getRootUser(), getRootPassword());
}
+ LOG.info("Starting {} initialization", getClass().getSimpleName());
+ SqlLine sqlLine = new SqlLine();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
sqlLine.setOutputStream(new PrintStream(out));
+ sqlLine.setErrorStream(new PrintStream(out));
System.setProperty("sqlline.silent", "true");
-
- SqlLine.Status status = sqlLine.begin(SQLLineCmdBuild(sqlScriptFile),
null, false);
- if (LOG.isDebugEnabled() && outputForLog != null) {
- LOG.debug("Received following output from Sqlline:");
- LOG.debug(outputForLog.toString("UTF-8"));
- }
+ SqlLine.Status status = sqlLine.begin(SQLLineCmdBuild(script), null,
false);
+ LOG.debug("Printing output from SQLLine:");
+ LOG.debug(out.toString());
if (status != SqlLine.Status.OK) {
- throw new IOException("external database script failed, errorcode
" + status);
+ throw new RuntimeException("Database script " + script + " failed
with status " + status);
}
+ LOG.info("Completed {} initialization", getClass().getSimpleName());
}
-
- public void execute(String script) throws IOException, SQLException,
ClassNotFoundException {
- testConnectionToExternalDB();
- LOG.info("Starting external database initialization to " +
this.externalDBType);
-
- try {
- LOG.info("Initialization script " + script);
- execSql(script);
- LOG.info("Initialization script completed in external database");
-
- } catch (IOException e) {
- throw new IOException("initialization in external database
FAILED!");
- }
-
- }
-}
\ No newline at end of file
+}
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/MSSQLServer.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/MSSQLServer.java
new file mode 100644
index 0000000..61dc014
--- /dev/null
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/MSSQLServer.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.externalDB;
+
+public class MSSQLServer extends AbstractExternalDB {
+
+ @Override
+ public String getRootUser() {
+ return "sa";
+ }
+
+ @Override
+ public String getRootPassword() {
+ return "Its-a-s3cret";
+ }
+
+ @Override
+ public String getJdbcUrl() {
+ return "jdbc:sqlserver://" + getContainerHostAddress() + ":1433";
+ }
+
+ @Override
+ public String getJdbcDriver() {
+ return "com.microsoft.sqlserver.jdbc.SQLServerDriver";
+ }
+
+ @Override
+ public String getDockerImageName() {
+ return "mcr.microsoft.com/mssql/server:2019-latest";
+ }
+
+ @Override
+ public String[] getDockerAdditionalArgs() {
+ return new String[] { "-p", "1433:1433", "-e", "ACCEPT_EULA=Y", "-e",
"SA_PASSWORD=" + getRootPassword(), "-d" };
+ }
+
+ @Override
+ public boolean isContainerReady(ProcessResults pr) {
+ return pr.stdout
+ .contains("Recovery is complete. This is an informational message
only. No user action is required.");
+ }
+}
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/MariaDB.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/MariaDB.java
new file mode 100644
index 0000000..c0d4bde
--- /dev/null
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/MariaDB.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.externalDB;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class MariaDB extends AbstractExternalDB {
+
+ @Override
+ public String getRootUser() {
+ return "root";
+ }
+
+ @Override
+ public String getJdbcUrl() {
+ return "jdbc:mariadb://" + getContainerHostAddress() + ":3309/" + dbName;
+ }
+
+ public String getJdbcDriver() {
+ return "org.mariadb.jdbc.Driver";
+ }
+
+ public String getDockerImageName() { return "mariadb:10.2"; }
+
+ public String[] getDockerAdditionalArgs() {
+ return new String[] {"-p", "3309:3306",
+ "-e", "MARIADB_ROOT_PASSWORD=" + getRootPassword(),
+ "-e", "MARIADB_DATABASE=" + dbName,
+ "-d"
+ };
+ }
+
+ public boolean isContainerReady(ProcessResults pr) {
+ Pattern pat = Pattern.compile("ready for connections");
+ Matcher m = pat.matcher(pr.stderr);
+ return m.find() && m.find();
+ }
+}
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/MySQLExternalDB.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/MySQLExternalDB.java
index 45668d9..ce7f3aa 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/MySQLExternalDB.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/MySQLExternalDB.java
@@ -21,34 +21,35 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
- * MySQLExternalDB is a extension of abstractExternalDB
* Designed for MySQL external database connection
*/
public class MySQLExternalDB extends AbstractExternalDB {
public MySQLExternalDB() {
- super("mysql");
- setJdbcUrl(getContainerHostAddress());
- setJdbcDriver();
}
- public void setJdbcUrl(String hostAddress) {
- this.url = "jdbc:mysql://" + hostAddress + ":3306/" + dbName;
+ @Override
+ public String getRootUser() {
+ return "root";
}
- public void setJdbcDriver() {
- this.driver = "org.mariadb.jdbc.Driver";
+ @Override
+ public String getJdbcUrl() {
+ return "jdbc:mysql://" + getContainerHostAddress() + ":3306/" + dbName;
}
- public String getDockerImageName() { return "mariadb:5.5"; }
+ public String getJdbcDriver() {
+ return "com.mysql.jdbc.Driver";
+ }
+
+ public String getDockerImageName() { return "mysql:5.7"; }
public String[] getDockerAdditionalArgs() {
- return buildArray("-p", "3306:3306",
- "-e", "MYSQL_ROOT_PASSWORD=its-a-secret",
- "-e", "MYSQL_USER=" + userName,
- "-e", "MYSQL_PASSWORD=" + password,
+ return new String[] {"-p", "3306:3306",
+ "-e", "MYSQL_ROOT_PASSWORD=" + getRootPassword(),
"-e", "MYSQL_DATABASE=" + dbName,
- "-d");
+ "-d"
+ };
}
public boolean isContainerReady(ProcessResults pr) {
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/Oracle.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/Oracle.java
new file mode 100644
index 0000000..34961c7
--- /dev/null
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/Oracle.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.externalDB;
+
+public class Oracle extends AbstractExternalDB {
+ @Override
+ public String getRootUser() {
+ return "SYS as SYSDBA";
+ }
+
+ @Override
+ public String getRootPassword() {
+ return "oracle";
+ }
+
+ @Override
+ public String getJdbcUrl() {
+ return "jdbc:oracle:thin:@//" + getContainerHostAddress() + ":1521/xe";
+ }
+
+ @Override
+ public String getJdbcDriver() {
+ return "oracle.jdbc.OracleDriver";
+ }
+
+ @Override
+ protected String getDockerImageName() {
+ return "pvargacl/oracle-xe-18.4.0";
+ }
+
+ @Override
+ protected String[] getDockerAdditionalArgs() {
+ return new String[] { "-p", "1521:1521", "-d" };
+ }
+
+ @Override
+ public boolean isContainerReady(ProcessResults pr) {
+ return pr.stdout.contains("DATABASE IS READY TO USE!");
+ }
+
+}
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/PostgresExternalDB.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/PostgresExternalDB.java
index 9365cf1..eaa427b 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/PostgresExternalDB.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/externalDB/PostgresExternalDB.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hive.ql.externalDB;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
@@ -30,17 +28,14 @@ import java.net.Socket;
public class PostgresExternalDB extends AbstractExternalDB {
public PostgresExternalDB() {
- super("postgres");
- setJdbcUrl(getContainerHostAddress());
- setJdbcDriver();
}
- public void setJdbcUrl(String hostAddress) {
- this.url = "jdbc:postgresql://" + hostAddress + ":5432/" + dbName;
+ public String getJdbcUrl() {
+ return "jdbc:postgresql://" + getContainerHostAddress() + ":5432/" +
dbName;
}
- public void setJdbcDriver() {
- this.driver = "org.postgresql.Driver";
+ public String getJdbcDriver() {
+ return "org.postgresql.Driver";
}
public String getDockerImageName() {
@@ -48,7 +43,12 @@ public class PostgresExternalDB extends AbstractExternalDB {
}
public String[] getDockerAdditionalArgs() {
- return buildArray("-p", "5432:5432", "-e",
"POSTGRES_PASSWORD=its-a-secret", "-d");
+ return new String[] {"-p", "5432:5432",
+ "-e", "POSTGRES_PASSWORD=" + getRootPassword(),
+ "-e", "POSTGRES_USER=" + getRootUser(),
+ "-e", "POSTGRES_DB=" + dbName,
+ "-d"
+ };
}
public boolean isContainerReady(ProcessResults pr) {
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestDatabaseHandler.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestDatabaseHandler.java
new file mode 100644
index 0000000..1c1e7db
--- /dev/null
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestDatabaseHandler.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.qoption;
+
+import org.apache.hadoop.hive.ql.QTestUtil;
+import org.apache.hadoop.hive.ql.externalDB.AbstractExternalDB;
+import org.apache.hadoop.hive.ql.externalDB.MSSQLServer;
+import org.apache.hadoop.hive.ql.externalDB.MariaDB;
+import org.apache.hadoop.hive.ql.externalDB.MySQLExternalDB;
+import org.apache.hadoop.hive.ql.externalDB.Oracle;
+import org.apache.hadoop.hive.ql.externalDB.PostgresExternalDB;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.EnumMap;
+import java.util.Map;
+
+/**
+ * An option handler for spinning (resp. stopping) databases before (resp.
after) running a test.
+ *
+ * Syntax: qt:database:DatabaseType:path_to_init_script
+ *
+ * The database type ({@link DatabaseType}) is obligatory argument. The
initialization script can be omitted.
+ *
+ * Current limitations:
+ * <ol>
+ * <li>Only one test/file per run</li>
+ * <li>Does not support parallel execution</li>
+ * <li>Cannot instantiate more than one database of the same type per
test</li>
+ * </ol>
+ */
+public class QTestDatabaseHandler implements QTestOptionHandler {
+ private static final Logger LOG =
LoggerFactory.getLogger(QTestDatabaseHandler.class);
+
+ private enum DatabaseType {
+ POSTGRES {
+ @Override
+ AbstractExternalDB create() {
+ return new PostgresExternalDB();
+ }
+ }, MYSQL {
+ @Override
+ AbstractExternalDB create() {
+ return new MySQLExternalDB();
+ }
+ }, MARIADB {
+ @Override
+ AbstractExternalDB create() {
+ return new MariaDB();
+ }
+ }, MSSQL {
+ @Override
+ AbstractExternalDB create() {
+ return new MSSQLServer();
+ }
+ }, ORACLE {
+ @Override
+ AbstractExternalDB create() {
+ return new Oracle();
+ }
+ };
+
+ abstract AbstractExternalDB create();
+ }
+
+ private final Map<DatabaseType, String> databaseToScript = new
EnumMap<>(DatabaseType.class);
+
+ @Override
+ public void processArguments(String arguments) {
+ String[] args = arguments.split(":");
+ if (args.length == 0) {
+ throw new IllegalArgumentException("No arguments provided");
+ }
+ if (args.length > 2) {
+ throw new IllegalArgumentException(
+ "Too many arguments. Expected {dbtype:script}. Found: " +
Arrays.toString(args));
+ }
+ DatabaseType dbType = DatabaseType.valueOf(args[0].toUpperCase());
+ String initScript = args.length == 2 ? args[1] : "";
+ if (databaseToScript.containsKey(dbType)) {
+ throw new IllegalArgumentException(dbType + " database is already
defined in this file.");
+ }
+ databaseToScript.put(dbType, initScript);
+ }
+
+ @Override
+ public void beforeTest(QTestUtil qt) throws Exception {
+ if (databaseToScript.isEmpty()) {
+ return;
+ }
+ for (Map.Entry<DatabaseType, String> dbEntry :
databaseToScript.entrySet()) {
+ String scriptsDir = QTestUtil.getScriptsDir(qt.getConf());
+ Path dbScript = Paths.get(scriptsDir, dbEntry.getValue());
+ AbstractExternalDB db = dbEntry.getKey().create();
+ db.launchDockerContainer();
+ if (Files.exists(dbScript)) {
+ db.execute(dbScript.toString());
+ } else {
+ LOG.warn("Initialization script {} not found", dbScript);
+ }
+ }
+ }
+
+ @Override
+ public void afterTest(QTestUtil qt) throws Exception {
+ if (databaseToScript.isEmpty()) {
+ return;
+ }
+ for (Map.Entry<DatabaseType, String> dbEntry :
databaseToScript.entrySet()) {
+ AbstractExternalDB db = dbEntry.getKey().create();
+ db.cleanupDockerContainer();
+ }
+ databaseToScript.clear();
+ }
+
+}
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/scheduled/QTestScheduledQueryCleaner.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/scheduled/QTestScheduledQueryCleaner.java
index de741f5..b9ac362 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/scheduled/QTestScheduledQueryCleaner.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/scheduled/QTestScheduledQueryCleaner.java
@@ -50,11 +50,11 @@ public class QTestScheduledQueryCleaner implements
QTestOptionHandler {
@Override
public void afterTest(QTestUtil qt) throws Exception {
- clearScheduledQueries(qt);
+ clearScheduledQueries(qt.getConf());
}
- public void clearScheduledQueries(QTestUtil qt) {
+ private void clearScheduledQueries(HiveConf conf) {
if (System.getenv(QTestUtil.QTEST_LEAVE_FILES) != null) {
return;
}
@@ -63,10 +63,9 @@ public class QTestScheduledQueryCleaner implements
QTestOptionHandler {
ScheduledQueryMaintenanceRequest request = new
ScheduledQueryMaintenanceRequest();
request.setType(ScheduledQueryMaintenanceRequestType.DROP);
ScheduledQuery schq = new ScheduledQuery();
- schq.setScheduleKey(new ScheduledQueryKey(name,
qt.getConf().getVar(ConfVars.HIVE_SCHEDULED_QUERIES_NAMESPACE)));
+ schq.setScheduleKey(new ScheduledQueryKey(name,
conf.getVar(ConfVars.HIVE_SCHEDULED_QUERIES_NAMESPACE)));
request.setScheduledQuery(schq);
try {
- HiveConf conf = qt.getConf();
Hive db = Hive.get(conf); // propagate new conf to meta store
db.getMSC().scheduledQueryMaintenance(request);
diff --git a/pom.xml b/pom.xml
index 23d24c0..b4e6ca4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -182,7 +182,9 @@
<libthrift.version>0.14.1</libthrift.version>
<log4j2.version>2.13.2</log4j2.version>
<mariadb.version>2.5.0</mariadb.version>
+ <mysql.version>8.0.27</mysql.version>
<postgres.version>42.2.14</postgres.version>
+ <oracle.version>21.3.0.0</oracle.version>
<opencsv.version>2.3</opencsv.version>
<orc.version>1.6.9</orc.version>
<mockito-core.version>3.4.4</mockito-core.version>
diff --git a/ql/src/test/queries/clientpositive/dataconnector_mysql.q
b/ql/src/test/queries/clientpositive/dataconnector_mysql.q
index f2c77eb..edab15c 100644
--- a/ql/src/test/queries/clientpositive/dataconnector_mysql.q
+++ b/ql/src/test/queries/clientpositive/dataconnector_mysql.q
@@ -1,10 +1,11 @@
+--!qt:database:mysql:q_test_country_state_city_tables.sql
-- CREATE with comment
CREATE CONNECTOR mysql_qtest
TYPE 'mysql'
URL 'jdbc:mysql://localhost:3306/qtestDB'
COMMENT 'test connector'
WITH DCPROPERTIES (
-"hive.sql.dbcp.username"="qtestuser",
+"hive.sql.dbcp.username"="root",
"hive.sql.dbcp.password"="qtestpassword");
SHOW CONNECTORS;
diff --git a/ql/src/test/queries/clientpositive/qt_database_all.q
b/ql/src/test/queries/clientpositive/qt_database_all.q
new file mode 100644
index 0000000..9695187
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/qt_database_all.q
@@ -0,0 +1,86 @@
+--!qt:database:postgres:q_test_country_table.sql
+--!qt:database:mysql:q_test_state_table.sql
+--!qt:database:mariadb:q_test_city_table.sql
+--!qt:database:mssql:q_test_author_table.sql
+--!qt:database:oracle:q_test_book_table.sql
+CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "POSTGRES",
+ "hive.sql.jdbc.driver" = "org.postgresql.Driver",
+ "hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
+ "hive.sql.dbcp.username" = "qtestuser",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "country"
+ );
+SELECT * FROM country;
+
+CREATE EXTERNAL TABLE state
+(
+ name varchar(255),
+ country int
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "com.mysql.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mysql://localhost:3306/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "state"
+ );
+SELECT * FROM state;
+
+CREATE EXTERNAL TABLE city
+(
+ name varchar(255),
+ state int
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "org.mariadb.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mariadb://localhost:3309/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "city"
+);
+SELECT * FROM city;
+
+CREATE EXTERNAL TABLE author
+(
+ id int,
+ fname varchar(20),
+ lname varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MSSQL",
+ "hive.sql.jdbc.driver" = "com.microsoft.sqlserver.jdbc.SQLServerDriver",
+ "hive.sql.jdbc.url" = "jdbc:sqlserver://localhost:1433",
+ "hive.sql.dbcp.username" = "sa",
+ "hive.sql.dbcp.password" = "Its-a-s3cret",
+ "hive.sql.table" = "author"
+ );
+SELECT * FROM author;
+
+CREATE EXTERNAL TABLE book
+(
+ id int,
+ title varchar(100),
+ author int
+)
+ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+ TBLPROPERTIES (
+ "hive.sql.database.type" = "ORACLE",
+ "hive.sql.jdbc.driver" = "oracle.jdbc.OracleDriver",
+ "hive.sql.jdbc.url" = "jdbc:oracle:thin:@//localhost:1521/xe",
+ "hive.sql.dbcp.username" = "SYS as SYSDBA",
+ "hive.sql.dbcp.password" = "oracle",
+ "hive.sql.table" = "BOOK"
+ );
+SELECT * FROM book;
diff --git a/ql/src/test/queries/clientpositive/qt_database_mariadb.q
b/ql/src/test/queries/clientpositive/qt_database_mariadb.q
new file mode 100644
index 0000000..5db9fd6
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/qt_database_mariadb.q
@@ -0,0 +1,17 @@
+--!qt:database:mariadb:q_test_country_table.sql
+CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "org.mariadb.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mariadb://localhost:3309/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "country"
+ );
+-- hive.sql.database.type above is not MARIADB cause at the moment it doesn't
exist in org.apache.hive.storage.jdbc.conf.DatabaseType
+SELECT * FROM country;
diff --git a/ql/src/test/queries/clientpositive/qt_database_mssql.q
b/ql/src/test/queries/clientpositive/qt_database_mssql.q
new file mode 100644
index 0000000..7240bb9
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/qt_database_mssql.q
@@ -0,0 +1,16 @@
+--!qt:database:mssql:q_test_country_table.sql
+CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MSSQL",
+ "hive.sql.jdbc.driver" = "com.microsoft.sqlserver.jdbc.SQLServerDriver",
+ "hive.sql.jdbc.url" = "jdbc:sqlserver://localhost:1433",
+ "hive.sql.dbcp.username" = "sa",
+ "hive.sql.dbcp.password" = "Its-a-s3cret",
+ "hive.sql.table" = "country"
+ );
+SELECT * FROM country;
diff --git a/ql/src/test/queries/clientpositive/qt_database_mysql.q
b/ql/src/test/queries/clientpositive/qt_database_mysql.q
new file mode 100644
index 0000000..9803df8
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/qt_database_mysql.q
@@ -0,0 +1,16 @@
+--!qt:database:mysql:q_test_country_table.sql
+CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "com.mysql.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mysql://localhost:3306/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "country"
+ );
+SELECT * FROM country;
diff --git a/ql/src/test/queries/clientpositive/qt_database_oracle.q
b/ql/src/test/queries/clientpositive/qt_database_oracle.q
new file mode 100644
index 0000000..8ec6beb
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/qt_database_oracle.q
@@ -0,0 +1,17 @@
+--!qt:database:oracle:q_test_country_table.sql
+CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "ORACLE",
+ "hive.sql.jdbc.driver" = "oracle.jdbc.OracleDriver",
+ "hive.sql.jdbc.url" = "jdbc:oracle:thin:@//localhost:1521/xe",
+ "hive.sql.dbcp.username" = "SYS as SYSDBA",
+ "hive.sql.dbcp.password" = "oracle",
+ "hive.sql.table" = "COUNTRY"
+);
+-- Case sensitivity is important in the default conf of Oracle. Notice the
capital letters in hive.sql.table
+SELECT * FROM country;
diff --git a/ql/src/test/queries/clientpositive/qt_database_postgres.q
b/ql/src/test/queries/clientpositive/qt_database_postgres.q
new file mode 100644
index 0000000..afc89f9
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/qt_database_postgres.q
@@ -0,0 +1,16 @@
+--!qt:database:postgres:q_test_country_table.sql
+CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+ TBLPROPERTIES (
+ "hive.sql.database.type" = "POSTGRES",
+ "hive.sql.jdbc.driver" = "org.postgresql.Driver",
+ "hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
+ "hive.sql.dbcp.username" = "qtestuser",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "country"
+ );
+SELECT * FROM country;
diff --git a/ql/src/test/results/clientpositive/llap/dataconnector_mysql.q.out
b/ql/src/test/results/clientpositive/llap/dataconnector_mysql.q.out
index 794fd2d..c771d29 100644
--- a/ql/src/test/results/clientpositive/llap/dataconnector_mysql.q.out
+++ b/ql/src/test/results/clientpositive/llap/dataconnector_mysql.q.out
@@ -3,7 +3,7 @@ TYPE 'mysql'
URL 'jdbc:mysql://localhost:3306/qtestDB'
COMMENT 'test connector'
WITH DCPROPERTIES (
-"hive.sql.dbcp.username"="qtestuser",
+"hive.sql.dbcp.username"="root",
"hive.sql.dbcp.password"="qtestpassword")
PREHOOK: type: CREATEDATACONNECTOR
PREHOOK: Output: connector:mysql_qtest
@@ -12,7 +12,7 @@ TYPE 'mysql'
URL 'jdbc:mysql://localhost:3306/qtestDB'
COMMENT 'test connector'
WITH DCPROPERTIES (
-"hive.sql.dbcp.username"="qtestuser",
+"hive.sql.dbcp.username"="root",
"hive.sql.dbcp.password"="qtestpassword")
POSTHOOK: type: CREATEDATACONNECTOR
POSTHOOK: Output: connector:mysql_qtest
diff --git a/ql/src/test/results/clientpositive/llap/qt_database_all.q.out
b/ql/src/test/results/clientpositive/llap/qt_database_all.q.out
new file mode 100644
index 0000000..2458c31
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/qt_database_all.q.out
@@ -0,0 +1,235 @@
+PREHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "POSTGRES",
+ "hive.sql.jdbc.driver" = "org.postgresql.Driver",
+ "hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
+ "hive.sql.dbcp.username" = "qtestuser",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "country"
+ )
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@country
+POSTHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "POSTGRES",
+ "hive.sql.jdbc.driver" = "org.postgresql.Driver",
+ "hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
+ "hive.sql.dbcp.username" = "qtestuser",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "country"
+ )
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@country
+PREHOOK: query: SELECT * FROM country
+PREHOOK: type: QUERY
+PREHOOK: Input: default@country
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM country
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@country
+#### A masked pattern was here ####
+1 India
+2 Russia
+3 USA
+PREHOOK: query: CREATE EXTERNAL TABLE state
+(
+ name varchar(255),
+ country int
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "com.mysql.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mysql://localhost:3306/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "state"
+ )
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@state
+POSTHOOK: query: CREATE EXTERNAL TABLE state
+(
+ name varchar(255),
+ country int
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "com.mysql.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mysql://localhost:3306/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "state"
+ )
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@state
+PREHOOK: query: SELECT * FROM state
+PREHOOK: type: QUERY
+PREHOOK: Input: default@state
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM state
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@state
+#### A masked pattern was here ####
+Maharashtra 1
+Madhya Pradesh 1
+Moscow 3
+Something 4
+Florida 4
+Texas 4
+PREHOOK: query: CREATE EXTERNAL TABLE city
+(
+ name varchar(255),
+ state int
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "org.mariadb.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mariadb://localhost:3309/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "city"
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@city
+POSTHOOK: query: CREATE EXTERNAL TABLE city
+(
+ name varchar(255),
+ state int
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "org.mariadb.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mariadb://localhost:3309/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "city"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@city
+PREHOOK: query: SELECT * FROM city
+PREHOOK: type: QUERY
+PREHOOK: Input: default@city
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM city
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@city
+#### A masked pattern was here ####
+Mumbai 1
+Pune 1
+Bhopal 2
+Indore 2
+Klin 3
+Los Angeles 5
+Plant City 5
+Arlington 6
+PREHOOK: query: CREATE EXTERNAL TABLE author
+(
+ id int,
+ fname varchar(20),
+ lname varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MSSQL",
+ "hive.sql.jdbc.driver" = "com.microsoft.sqlserver.jdbc.SQLServerDriver",
+ "hive.sql.jdbc.url" = "jdbc:sqlserver://localhost:1433",
+ "hive.sql.dbcp.username" = "sa",
+ "hive.sql.dbcp.password" = "Its-a-s3cret",
+ "hive.sql.table" = "author"
+ )
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@author
+POSTHOOK: query: CREATE EXTERNAL TABLE author
+(
+ id int,
+ fname varchar(20),
+ lname varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MSSQL",
+ "hive.sql.jdbc.driver" = "com.microsoft.sqlserver.jdbc.SQLServerDriver",
+ "hive.sql.jdbc.url" = "jdbc:sqlserver://localhost:1433",
+ "hive.sql.dbcp.username" = "sa",
+ "hive.sql.dbcp.password" = "Its-a-s3cret",
+ "hive.sql.table" = "author"
+ )
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@author
+PREHOOK: query: SELECT * FROM author
+PREHOOK: type: QUERY
+PREHOOK: Input: default@author
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM author
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@author
+#### A masked pattern was here ####
+1 Victor Hugo
+2 Alexandre Dumas
+PREHOOK: query: CREATE EXTERNAL TABLE book
+(
+ id int,
+ title varchar(100),
+ author int
+)
+ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+ TBLPROPERTIES (
+ "hive.sql.database.type" = "ORACLE",
+ "hive.sql.jdbc.driver" = "oracle.jdbc.OracleDriver",
+ "hive.sql.jdbc.url" = "jdbc:oracle:thin:@//localhost:1521/xe",
+ "hive.sql.dbcp.username" = "SYS as SYSDBA",
+ "hive.sql.dbcp.password" = "oracle",
+ "hive.sql.table" = "BOOK"
+ )
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@book
+POSTHOOK: query: CREATE EXTERNAL TABLE book
+(
+ id int,
+ title varchar(100),
+ author int
+)
+ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+ TBLPROPERTIES (
+ "hive.sql.database.type" = "ORACLE",
+ "hive.sql.jdbc.driver" = "oracle.jdbc.OracleDriver",
+ "hive.sql.jdbc.url" = "jdbc:oracle:thin:@//localhost:1521/xe",
+ "hive.sql.dbcp.username" = "SYS as SYSDBA",
+ "hive.sql.dbcp.password" = "oracle",
+ "hive.sql.table" = "BOOK"
+ )
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@book
+PREHOOK: query: SELECT * FROM book
+PREHOOK: type: QUERY
+PREHOOK: Input: default@book
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM book
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@book
+#### A masked pattern was here ####
+1 Les Miserables 1
+2 The Count Of Monte Cristo 2
diff --git a/ql/src/test/results/clientpositive/llap/qt_database_mariadb.q.out
b/ql/src/test/results/clientpositive/llap/qt_database_mariadb.q.out
new file mode 100644
index 0000000..f00db08
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/qt_database_mariadb.q.out
@@ -0,0 +1,45 @@
+PREHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "org.mariadb.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mariadb://localhost:3309/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "country"
+ )
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@country
+POSTHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "org.mariadb.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mariadb://localhost:3309/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "country"
+ )
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@country
+PREHOOK: query: SELECT * FROM country
+PREHOOK: type: QUERY
+PREHOOK: Input: default@country
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM country
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@country
+#### A masked pattern was here ####
+1 India
+2 Russia
+3 USA
diff --git a/ql/src/test/results/clientpositive/llap/qt_database_mssql.q.out
b/ql/src/test/results/clientpositive/llap/qt_database_mssql.q.out
new file mode 100644
index 0000000..a125dd8
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/qt_database_mssql.q.out
@@ -0,0 +1,45 @@
+PREHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MSSQL",
+ "hive.sql.jdbc.driver" = "com.microsoft.sqlserver.jdbc.SQLServerDriver",
+ "hive.sql.jdbc.url" = "jdbc:sqlserver://localhost:1433",
+ "hive.sql.dbcp.username" = "sa",
+ "hive.sql.dbcp.password" = "Its-a-s3cret",
+ "hive.sql.table" = "country"
+ )
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@country
+POSTHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MSSQL",
+ "hive.sql.jdbc.driver" = "com.microsoft.sqlserver.jdbc.SQLServerDriver",
+ "hive.sql.jdbc.url" = "jdbc:sqlserver://localhost:1433",
+ "hive.sql.dbcp.username" = "sa",
+ "hive.sql.dbcp.password" = "Its-a-s3cret",
+ "hive.sql.table" = "country"
+ )
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@country
+PREHOOK: query: SELECT * FROM country
+PREHOOK: type: QUERY
+PREHOOK: Input: default@country
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM country
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@country
+#### A masked pattern was here ####
+1 India
+2 Russia
+3 USA
diff --git a/ql/src/test/results/clientpositive/llap/qt_database_mysql.q.out
b/ql/src/test/results/clientpositive/llap/qt_database_mysql.q.out
new file mode 100644
index 0000000..b5f1c86
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/qt_database_mysql.q.out
@@ -0,0 +1,45 @@
+PREHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "com.mysql.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mysql://localhost:3306/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "country"
+ )
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@country
+POSTHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "MYSQL",
+ "hive.sql.jdbc.driver" = "com.mysql.jdbc.Driver",
+ "hive.sql.jdbc.url" = "jdbc:mysql://localhost:3306/qtestDB",
+ "hive.sql.dbcp.username" = "root",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "country"
+ )
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@country
+PREHOOK: query: SELECT * FROM country
+PREHOOK: type: QUERY
+PREHOOK: Input: default@country
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM country
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@country
+#### A masked pattern was here ####
+1 India
+2 Russia
+3 USA
diff --git a/ql/src/test/results/clientpositive/llap/qt_database_oracle.q.out
b/ql/src/test/results/clientpositive/llap/qt_database_oracle.q.out
new file mode 100644
index 0000000..d63d30f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/qt_database_oracle.q.out
@@ -0,0 +1,45 @@
+PREHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "ORACLE",
+ "hive.sql.jdbc.driver" = "oracle.jdbc.OracleDriver",
+ "hive.sql.jdbc.url" = "jdbc:oracle:thin:@//localhost:1521/xe",
+ "hive.sql.dbcp.username" = "SYS as SYSDBA",
+ "hive.sql.dbcp.password" = "oracle",
+ "hive.sql.table" = "COUNTRY"
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@country
+POSTHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+TBLPROPERTIES (
+ "hive.sql.database.type" = "ORACLE",
+ "hive.sql.jdbc.driver" = "oracle.jdbc.OracleDriver",
+ "hive.sql.jdbc.url" = "jdbc:oracle:thin:@//localhost:1521/xe",
+ "hive.sql.dbcp.username" = "SYS as SYSDBA",
+ "hive.sql.dbcp.password" = "oracle",
+ "hive.sql.table" = "COUNTRY"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@country
+PREHOOK: query: SELECT * FROM country
+PREHOOK: type: QUERY
+PREHOOK: Input: default@country
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM country
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@country
+#### A masked pattern was here ####
+1 India
+2 Russia
+3 USA
diff --git a/ql/src/test/results/clientpositive/llap/qt_database_postgres.q.out
b/ql/src/test/results/clientpositive/llap/qt_database_postgres.q.out
new file mode 100644
index 0000000..77ab839
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/qt_database_postgres.q.out
@@ -0,0 +1,45 @@
+PREHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+ TBLPROPERTIES (
+ "hive.sql.database.type" = "POSTGRES",
+ "hive.sql.jdbc.driver" = "org.postgresql.Driver",
+ "hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
+ "hive.sql.dbcp.username" = "qtestuser",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "country"
+ )
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@country
+POSTHOOK: query: CREATE EXTERNAL TABLE country
+(
+ id int,
+ name varchar(20)
+)
+ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
+ TBLPROPERTIES (
+ "hive.sql.database.type" = "POSTGRES",
+ "hive.sql.jdbc.driver" = "org.postgresql.Driver",
+ "hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
+ "hive.sql.dbcp.username" = "qtestuser",
+ "hive.sql.dbcp.password" = "qtestpassword",
+ "hive.sql.table" = "country"
+ )
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@country
+PREHOOK: query: SELECT * FROM country
+PREHOOK: type: QUERY
+PREHOOK: Input: default@country
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM country
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@country
+#### A masked pattern was here ####
+1 India
+2 Russia
+3 USA
