Repository: hive Updated Branches: refs/heads/master 75c31e7ab -> 1dccdeada
HIVE-17706: Add a possibility to run the BeeLine tests on the default database (Peter Vary, reviewed by Barna Zsombor Klara) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1dccdead Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1dccdead Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1dccdead Branch: refs/heads/master Commit: 1dccdeadaba35a0206b17d512ef1cf9aced374b8 Parents: 75c31e7 Author: Peter Vary <[email protected]> Authored: Mon Oct 9 12:09:06 2017 +0200 Committer: Peter Vary <[email protected]> Committed: Mon Oct 9 12:09:06 2017 +0200 ---------------------------------------------------------------------- .../hive/cli/control/CoreBeeLineDriver.java | 6 +- .../java/org/apache/hive/beeline/QFile.java | 65 ++++++--- .../apache/hive/beeline/QFileBeeLineClient.java | 138 +++++++++++++++---- 3 files changed, 159 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/1dccdead/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java index b44ffbd..66ccd6f 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java @@ -49,7 +49,7 @@ public class CoreBeeLineDriver extends CliAdapter { private final File testDataDirectory; private final File testScriptDirectory; private boolean overwrite = false; - private boolean rewriteSourceTables = true; + private boolean useSharedDatabase = false; private MiniHS2 miniHS2; private QFileClientBuilder clientBuilder; private QFileBuilder fileBuilder; @@ -111,7 +111,7 @@ public class CoreBeeLineDriver extends CliAdapter { public void beforeClass() throws Exception { overwrite = getBooleanPropertyValue("test.output.overwrite", Boolean.FALSE); - rewriteSourceTables = getBooleanPropertyValue("test.rewrite.source.tables", Boolean.TRUE); + useSharedDatabase = getBooleanPropertyValue("test.beeline.shared.database", Boolean.FALSE); String beeLineUrl = System.getProperty("test.beeline.url"); if (StringUtils.isEmpty(beeLineUrl)) { @@ -132,7 +132,7 @@ public class CoreBeeLineDriver extends CliAdapter { .setLogDirectory(logDirectory) .setQueryDirectory(queryDirectory) .setResultsDirectory(resultsDirectory) - .setRewriteSourceTables(rewriteSourceTables) + .setUseSharedDatabase(useSharedDatabase) .setComparePortable(comparePortable); runInfraScript(initScript, new File(logDirectory, "init.beeline"), http://git-wip-us.apache.org/repos/asf/hive/blob/1dccdead/itests/util/src/main/java/org/apache/hive/beeline/QFile.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java index 38b0d91..21be8b0 100644 --- a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java +++ b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java @@ -81,7 +81,7 @@ public final class QFile { private static RegexFilterSet staticFilterSet = getStaticFilterSet(); private static RegexFilterSet portableFilterSet = getPortableFilterSet(); private RegexFilterSet specificFilterSet; - private boolean rewriteSourceTables; + private boolean useSharedDatabase; private Converter converter; private boolean comparePortable; @@ -91,6 +91,7 @@ public final class QFile { return name; } + public String getDatabaseName() { return databaseName; } @@ -127,6 +128,10 @@ public final class QFile { return converter; } + public boolean isUseSharedDatabase() { + return useSharedDatabase; + } + public String getDebugHint() { return String.format(DEBUG_HINT, inputFile, rawOutputFile, outputFile, expectedOutputFile, logFile, beforeExecuteLogFile, afterExecuteLogFile, @@ -134,13 +139,13 @@ public final class QFile { } /** - * Filters the sql commands if necessary. + * Filters the sql commands if necessary - eg. not using the shared database. * @param commands The array of the sql commands before filtering * @return The filtered array of the sql command strings * @throws IOException File read error */ public String[] filterCommands(String[] commands) throws IOException { - if (rewriteSourceTables) { + if (!useSharedDatabase) { for (int i=0; i<commands.length; i++) { if (USE_PATTERN.matcher(commands[i]).matches()) { System.err.println(String.format(USE_COMMAND_WARNING, inputFile, commands[i])); @@ -177,8 +182,8 @@ public final class QFile { */ private String revertReplaceTableNames(String source) { for (String table : srcTables) { - source = source.replaceAll("(?is)(?<!name:?|alias:?)(\\s+)default\\." + table - + "([\\s;\\n\\)])", "$1" + table + "$2"); + source = source.replaceAll("(?is)(?<!name:?|alias:?)(\\s+)default\\.(" + table + + ")([\\s;\\n\\),])", "$1$2$3"); } return source; } @@ -199,18 +204,28 @@ public final class QFile { return source; } + /** + * Filters the generated output file + * @throws IOException + */ public void filterOutput() throws IOException { String output = FileUtils.readFileToString(rawOutputFile, "UTF-8"); if (comparePortable) { output = portableFilterSet.filter(output); } output = staticFilterSet.filter(specificFilterSet.filter(output)); - if (rewriteSourceTables) { + if (!useSharedDatabase) { output = sortInputOutput(revertReplaceTableNames(output)); } FileUtils.writeStringToFile(outputFile, output); } + /** + * Compare the filtered file with the expected golden file + * @return The comparison data + * @throws IOException If there is a problem accessing the golden or generated file + * @throws InterruptedException If there is a problem running the diff command + */ public QTestProcessExecResult compareResults() throws IOException, InterruptedException { if (!expectedOutputFile.exists()) { throw new IOException("Expected results file does not exist: " + expectedOutputFile); @@ -218,6 +233,10 @@ public final class QFile { return executeDiff(); } + /** + * Overwrite the golden file with the generated output + * @throws IOException If there is a problem accessing the golden or generated file + */ public void overwriteResults() throws IOException { FileUtils.copyFile(outputFile, expectedOutputFile); } @@ -316,8 +335,11 @@ public final class QFile { .addFilter(".*/tmp/.*\n", MASK_PATTERN) .addFilter(".*file:.*\n", MASK_PATTERN) .addFilter(".*file\\..*\n", MASK_PATTERN) + .addFilter(".*Location.*\n", MASK_PATTERN) + .addFilter(".*LOCATION '.*\n", MASK_PATTERN) .addFilter(".*Output:.*/data/files/.*\n", MASK_PATTERN) .addFilter(".*CreateTime.*\n", MASK_PATTERN) + .addFilter(".*last_modified_.*\n", MASK_PATTERN) .addFilter(".*transient_lastDdlTime.*\n", MASK_PATTERN) .addFilter(".*lastUpdateTime.*\n", MASK_PATTERN) .addFilter(".*lastAccessTime.*\n", MASK_PATTERN) @@ -349,7 +371,7 @@ public final class QFile { private File queryDirectory; private File logDirectory; private File resultsDirectory; - private boolean rewriteSourceTables; + private boolean useSharedDatabase; private boolean comparePortable; public QFileBuilder() { @@ -370,8 +392,8 @@ public final class QFile { return this; } - public QFileBuilder setRewriteSourceTables(boolean rewriteSourceTables) { - this.rewriteSourceTables = rewriteSourceTables; + public QFileBuilder setUseSharedDatabase(boolean useSharedDatabase) { + this.useSharedDatabase = useSharedDatabase; return this; } @@ -383,22 +405,27 @@ public final class QFile { public QFile getQFile(String name) throws IOException { QFile result = new QFile(); result.name = name; - result.databaseName = "test_db_" + name; + if (!useSharedDatabase) { + result.databaseName = "test_db_" + name.toLowerCase(); + result.specificFilterSet = new RegexFilterSet() + .addFilter("(PREHOOK|POSTHOOK): (Output|Input): database:" + result.databaseName + "\n", + "$1: $2: database:default\n") + .addFilter("(PREHOOK|POSTHOOK): (Output|Input): " + result.databaseName + "@", + "$1: $2: default@") + .addFilter("name(:?) " + result.databaseName + "\\.(.*)\n", "name$1 default.$2\n") + .addFilter("alias(:?) " + result.databaseName + "\\.(.*)\n", "alias$1 default.$2\n") + .addFilter("/" + result.databaseName + ".db/", "/"); + } else { + result.databaseName = "default"; + result.specificFilterSet = new RegexFilterSet(); + } result.inputFile = new File(queryDirectory, name + ".q"); result.rawOutputFile = new File(logDirectory, name + ".q.out.raw"); result.outputFile = new File(logDirectory, name + ".q.out"); result.logFile = new File(logDirectory, name + ".q.beeline"); result.beforeExecuteLogFile = new File(logDirectory, name + ".q.beforeExecute.log"); result.afterExecuteLogFile = new File(logDirectory, name + ".q.afterExecute.log"); - result.rewriteSourceTables = rewriteSourceTables; - result.specificFilterSet = new RegexFilterSet() - .addFilter("(PREHOOK|POSTHOOK): (Output|Input): database:" + result.databaseName + "\n", - "$1: $2: database:default\n") - .addFilter("(PREHOOK|POSTHOOK): (Output|Input): " + result.databaseName + "@", - "$1: $2: default@") - .addFilter("name(:?) " + result.databaseName + "\\.(.*)\n", "name$1 default.$2\n") - .addFilter("alias(:?) " + result.databaseName + "\\.(.*)\n", "alias$1 default.$2\n") - .addFilter("/" + result.databaseName + ".db/", "/"); + result.useSharedDatabase = useSharedDatabase; result.converter = Converter.NONE; String input = FileUtils.readFileToString(result.inputFile, "UTF-8"); if (input.contains("-- SORT_QUERY_RESULTS")) { http://git-wip-us.apache.org/repos/asf/hive/blob/1dccdead/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java b/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java index 2f91834..d84ff39 100644 --- a/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java +++ b/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java @@ -18,12 +18,19 @@ package org.apache.hive.beeline; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hive.beeline.ConvertedOutputFile.Converter; import java.io.File; import java.io.IOException; import java.io.PrintStream; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; import java.sql.SQLException; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; /** * QFile test client using BeeLine. It can be used to submit a list of command strings, or a QFile. @@ -32,6 +39,26 @@ public class QFileBeeLineClient implements AutoCloseable { private BeeLine beeLine; private PrintStream beelineOutputStream; private File logFile; + private String[] TEST_FIRST_COMMANDS = new String[] { + "!set outputformat tsv2", + "!set verbose false", + "!set silent true", + "!set showheader false", + "USE default;", + "SHOW TABLES;", + }; + private String[] TEST_SET_LOG_COMMANDS = new String[] { + "set hive.testing.short.logs=true;", + "set hive.testing.remove.logs=false;", + }; + private String[] TEST_RESET_COMMANDS = new String[] { + "set hive.testing.short.logs=false;", + "!set verbose true", + "!set silent false", + "!set showheader true", + "!set outputformat table", + "USE default;" + }; protected QFileBeeLineClient(String jdbcUrl, String jdbcDriver, String username, String password, File log) throws IOException { @@ -52,6 +79,47 @@ public class QFileBeeLineClient implements AutoCloseable { }); } + private Set<String> getDatabases() throws SQLException { + Set<String> databases = new HashSet<String>(); + + DatabaseMetaData metaData = beeLine.getDatabaseMetaData(); + // Get the databases + try (ResultSet schemasResultSet = metaData.getSchemas()) { + while (schemasResultSet.next()) { + databases.add(schemasResultSet.getString("TABLE_SCHEM")); + } + } + return databases; + } + + private Set<String> getTables() throws SQLException { + Set<String> tables = new HashSet<String>(); + + DatabaseMetaData metaData = beeLine.getDatabaseMetaData(); + // Get the tables in the default database + String[] types = new String[] {"TABLE"}; + try (ResultSet tablesResultSet = metaData.getTables(null, "default", "%", types)) { + while (tablesResultSet.next()) { + tables.add(tablesResultSet.getString("TABLE_NAME")); + } + } + return tables; + } + + private Set<String> getViews() throws SQLException { + Set<String> views = new HashSet<String>(); + + DatabaseMetaData metaData = beeLine.getDatabaseMetaData(); + // Get the tables in the default database + String[] types = new String[] {"VIEW"}; + try (ResultSet tablesResultSet = metaData.getTables(null, "default", "%", types)) { + while (tablesResultSet.next()) { + views.add(tablesResultSet.getString("TABLE_NAME")); + } + } + return views; + } + public void execute(String[] commands, File resultFile, Converter converter) throws Exception { beeLine.runCommands( @@ -69,39 +137,53 @@ public class QFileBeeLineClient implements AutoCloseable { } private void beforeExecute(QFile qFile) throws Exception { - execute( - new String[] { - "!set outputformat tsv2", - "!set verbose false", - "!set silent true", - "!set showheader false", - "USE default;", - "SHOW TABLES;", - "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;", - "CREATE DATABASE `" + qFile.getDatabaseName() + "`;", - "USE `" + qFile.getDatabaseName() + "`;", - "set hive.testing.short.logs=true;", - "set hive.testing.remove.logs=false;", - }, - qFile.getBeforeExecuteLogFile(), - Converter.NONE); + String[] commands = TEST_FIRST_COMMANDS; + + String[] extraCommands; + if (qFile.isUseSharedDatabase()) { + // If we are using a shared database, then remove not known databases, tables, views. + Set<String> dropCommands = getDatabases().stream() + .filter(database -> !database.equals("default")) + .map(database -> "DROP DATABASE `" + database + "` CASCADE;") + .collect(Collectors.toSet()); + + Set<String> srcTables = QTestUtil.getSrcTables(); + dropCommands.addAll(getTables().stream() + .filter(table -> !srcTables.contains(table)) + .map(table -> "DROP TABLE `" + table + "` PURGE;") + .collect(Collectors.toSet())); + + dropCommands.addAll(getViews().stream() + .map(view -> "DROP VIEW `" + view + "`;") + .collect(Collectors.toSet())); + extraCommands = dropCommands.toArray(new String[]{}); + } else { + // If we are using a test specific database, then we just drop the database, and recreate + extraCommands = new String[] { + "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;", + "CREATE DATABASE `" + qFile.getDatabaseName() + "`;", + "USE `" + qFile.getDatabaseName() + "`;" + }; + } + commands = ArrayUtils.addAll(commands, extraCommands); + commands = ArrayUtils.addAll(commands, TEST_SET_LOG_COMMANDS); + execute(commands, qFile.getBeforeExecuteLogFile(), Converter.NONE); beeLine.setIsTestMode(true); } private void afterExecute(QFile qFile) throws Exception { beeLine.setIsTestMode(false); - execute( - new String[] { - "set hive.testing.short.logs=false;", - "!set verbose true", - "!set silent false", - "!set showheader true", - "!set outputformat table", - "USE default;", - "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;", - }, - qFile.getAfterExecuteLogFile(), - Converter.NONE); + String[] commands = TEST_RESET_COMMANDS; + + if (!qFile.isUseSharedDatabase()) { + // If we are using a test specific database, then we just drop the database + String[] extraCommands = new String[] { + "DROP DATABASE IF EXISTS `" + qFile.getDatabaseName() + "` CASCADE;" + }; + commands = ArrayUtils.addAll(commands, extraCommands); + } + + execute(commands, qFile.getAfterExecuteLogFile(), Converter.NONE); } public void execute(QFile qFile) throws Exception {
