This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 82feb02 HIVE-21034: Add option to schematool to drop Hive databases
(Daniel Voros via Ashutosh Chauhan)
82feb02 is described below
commit 82feb0265f291065e6c7b26695c984bf9cb63cde
Author: Daniel Voros <[email protected]>
AuthorDate: Mon Apr 1 12:19:45 2019 +0200
HIVE-21034: Add option to schematool to drop Hive databases (Daniel Voros
via Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich <[email protected]>
---
.../tools/schematool/HiveSchemaHelper.java | 2 +-
.../tools/schematool/MetastoreSchemaTool.java | 14 ++-
.../tools/schematool/SchemaToolCommandLine.java | 13 +++
.../tools/schematool/SchemaToolTaskDrop.java | 130 +++++++++++++++++++++
.../tools/schematool/TestSchemaToolTaskDrop.java | 129 ++++++++++++++++++++
5 files changed, 282 insertions(+), 6 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/HiveSchemaHelper.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/HiveSchemaHelper.java
index caf9c31..cfec036 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/HiveSchemaHelper.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/HiveSchemaHelper.java
@@ -71,7 +71,7 @@ public class HiveSchemaHelper {
driver = driver == null ?
getValidConfVar(MetastoreConf.ConfVars.CONNECTION_DRIVER, conf) : driver;
if (printInfo) {
logAndPrintToStdout("Metastore connection URL:\t " + url);
- logAndPrintToStdout("Metastore Connection Driver :\t " + driver);
+ logAndPrintToStdout("Metastore connection Driver :\t " + driver);
logAndPrintToStdout("Metastore connection User:\t " + userName);
if (MetastoreConf.getBoolVar(conf,
MetastoreConf.ConfVars.HIVE_IN_TEST)) {
logAndPrintToStdout("Metastore connection Password:\t " + password);
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java
index 1517fa1..eafe0c6 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java
@@ -111,7 +111,7 @@ public class MetastoreSchemaTool {
this.metaStoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf,
metastoreHome, dbType);
// If the dbType is "hive", this is setting up the information schema in
Hive.
// We will set the default jdbc url and driver.
- // It is overriden by command line options if passed (-url and -driver
+ // It is overridden by command line options if passed (-url and -driver)
if (dbType.equalsIgnoreCase(HiveSchemaHelper.DB_HIVE)) {
this.url = HiveSchemaHelper.EMBEDDED_HS2_URL;
this.driver = HiveSchemaHelper.HIVE_JDBC_DRIVER;
@@ -437,6 +437,8 @@ public class MetastoreSchemaTool {
task = new SchemaToolTaskMoveTable();
} else if (cmdLine.hasOption("createUser")) {
task = new SchemaToolTaskCreateUser();
+ } else if (cmdLine.hasOption("dropAllDatabases")) {
+ task = new SchemaToolTaskDrop();
} else {
throw new HiveMetaException("No task defined!");
}
@@ -456,10 +458,12 @@ public class MetastoreSchemaTool {
logAndPrintToError("SQL Error code: " + ((SQLException)
t).getErrorCode());
}
}
- if (cmdLine.hasOption("verbose")) {
- e.printStackTrace();
- } else {
- logAndPrintToError("Use --verbose for detailed stacktrace.");
+ if (cmdLine != null) {
+ if (cmdLine.hasOption("verbose")) {
+ e.printStackTrace();
+ } else {
+ logAndPrintToError("Use --verbose for detailed stacktrace.");
+ }
}
logAndPrintToError("*** schemaTool failed ***");
return 1;
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java
index 84b85e9..6282078 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java
@@ -47,6 +47,9 @@ public class SchemaToolCommandLine {
.withDescription("Schema initialization to a version")
.create("initSchemaTo");
Option initOrUpgradeSchemaOpt = new Option("initOrUpgradeSchema",
"Initialize or upgrade schema to latest version");
+ Option dropDbOpt = new Option("dropAllDatabases", "Drop all Hive databases
(with CASCADE). " +
+ "This will remove all managed data!");
+ Option yesOpt = new Option("yes", "Don't ask for confirmation when using
-dropAllDatabases.");
Option validateOpt = new Option("validate", "Validate the database");
Option createCatalog = OptionBuilder
.hasArg()
@@ -77,6 +80,7 @@ public class SchemaToolCommandLine {
.addOption(upgradeOpt)
.addOption(upgradeFromOpt)
.addOption(initOpt)
+ .addOption(dropDbOpt)
.addOption(initToOpt)
.addOption(initOrUpgradeSchemaOpt)
.addOption(validateOpt)
@@ -185,6 +189,7 @@ public class SchemaToolCommandLine {
options.addOption(hiveUserOpt);
options.addOption(hivePasswdOpt);
options.addOption(hiveDbOpt);
+ options.addOption(yesOpt);
if (additionalOptions != null) options.addOptionGroup(additionalOptions);
return options;
@@ -276,6 +281,14 @@ public class SchemaToolCommandLine {
(cl.hasOption("fromDatabase") || cl.hasOption("toDatabase"))) {
printAndExit("fromDatabase and toDatabase may be set only for
moveTable");
}
+
+ if (cl.hasOption("dropAllDatabases") &&
!HiveSchemaHelper.DB_HIVE.equals(dbType)) {
+ printAndExit("dropAllDatabases can only be used with dbType=hive");
+ }
+
+ if (cl.hasOption("yes") && !cl.hasOption("dropAllDatabases")) {
+ printAndExit("yes can only be used with dropAllDatabases");
+ }
}
private void printAndExit(String reason) throws ParseException {
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolTaskDrop.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolTaskDrop.java
new file mode 100644
index 0000000..aa8ca94
--- /dev/null
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolTaskDrop.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.tools.schematool;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.Warehouse;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Scanner;
+
+/**
+ * {@link SchemaToolTaskDrop} drops all data from Hive. It invokes DROP TABLE
on all
+ * tables of the default database and DROP DATABASE CASCADE on all other
databases.
+ */
+public class SchemaToolTaskDrop extends SchemaToolTask {
+
+ @VisibleForTesting
+ boolean yes = false;
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cmdLine) {
+ if (cmdLine.hasOption("yes")) {
+ this.yes = true;
+ }
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ // Need to confirm unless it's a dry run or specified -yes
+ if (!schemaTool.isDryRun() && !this.yes) {
+ boolean confirmed = promptToConfirm();
+ if (!confirmed) {
+ System.out.println("Operation cancelled, exiting.");
+ return;
+ }
+ }
+
+ Connection conn = schemaTool.getConnectionToMetastore(true);
+ try {
+ try (Statement stmt = conn.createStatement()) {
+ final String def = Warehouse.DEFAULT_DATABASE_NAME;
+
+ // List databases
+ List<String> databases = new ArrayList<>();
+ try (ResultSet rs = stmt.executeQuery("SHOW DATABASES")) {
+ while (rs.next()) {
+ databases.add(rs.getString(1));
+ }
+ }
+
+ // Drop databases
+ for (String database : databases) {
+ // Don't try to drop 'default' database as it's not allowed
+ if (!def.equalsIgnoreCase(database)) {
+ if (schemaTool.isDryRun()) {
+ System.out.println("would drop database " + database);
+ } else {
+ logIfVerbose("dropping database " + database);
+ stmt.execute(String.format("DROP DATABASE `%s` CASCADE",
database));
+ }
+ }
+ }
+
+ // List tables in 'default' database
+ List<String> tables = new ArrayList<>();
+ try (ResultSet rs = stmt.executeQuery(String.format("SHOW TABLES IN
`%s`", def))) {
+ while (rs.next()) {
+ tables.add(rs.getString(1));
+ }
+ }
+
+ // Drop tables in 'default' database
+ for (String table : tables) {
+ if (schemaTool.isDryRun()) {
+ System.out.println("would drop table " + table);
+ } else {
+ logIfVerbose("dropping table " + table);
+ stmt.execute(String.format("DROP TABLE `%s`.`%s`", def, table));
+ }
+ }
+ }
+ } catch (SQLException se) {
+ throw new HiveMetaException("Failed to drop databases.", se);
+ }
+ }
+
+ /**
+ * Display "are you sure? y/n" on command line and return what the user has
chosen
+ * @return
+ */
+ private boolean promptToConfirm() {
+ System.out.print("This operation will delete ALL managed data in Hive. " +
+ "Are you sure you want to continue (y/[n])?");
+ Scanner scanner = new Scanner(System.in, "UTF-8");
+ String input = scanner.nextLine();
+
+ if ("y".equalsIgnoreCase(input) || "yes".equalsIgnoreCase(input)) {
+ return true;
+ }
+ return false;
+ }
+
+ private void logIfVerbose(String msg) {
+ if (schemaTool.isVerbose()) {
+ System.out.println(msg);
+ }
+ }
+}
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolTaskDrop.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolTaskDrop.java
new file mode 100644
index 0000000..271f83a
--- /dev/null
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolTaskDrop.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.tools.schematool;
+
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.when;
+
+/**
+ * Testing SchemaToolTaskDrop.
+ */
+public class TestSchemaToolTaskDrop {
+
+ private SchemaToolTaskDrop uut;
+
+ private Statement stmtMock;
+
+ private final InputStream stdin = System.in;
+
+ @Before
+ public void setUp() throws Exception {
+ uut = new SchemaToolTaskDrop();
+ uut.schemaTool = mock(MetastoreSchemaTool.class);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ System.setIn(stdin);
+ }
+
+ private void mockPromptWith(String answer) {
+ InputStream in = new ByteArrayInputStream((answer + "\n").getBytes());
+ System.setIn(in);
+ }
+
+ private void setUpTwoDatabases() throws Exception {
+ Connection connMock = mock(Connection.class);
+ stmtMock = mock(Statement.class);
+
+ // Return two databases: `mydb` and `default`
+ ResultSet databasesResult = mock(ResultSet.class);
+ when(databasesResult.next()).thenReturn(true, true, false);
+ when(databasesResult.getString(anyInt())).thenReturn("mydb",
Warehouse.DEFAULT_DATABASE_NAME);
+
+ // Return two tables: `table1` and `table2`
+ ResultSet tablesResult = mock(ResultSet.class);
+ when(tablesResult.next()).thenReturn(true, true, false);
+ when(tablesResult.getString(anyInt())).thenReturn("table1", "table2");
+
+ when(stmtMock.executeQuery(anyString())).thenReturn(databasesResult,
tablesResult);
+ when(connMock.createStatement()).thenReturn(stmtMock);
+
+
when(uut.schemaTool.getConnectionToMetastore(anyBoolean())).thenReturn(connMock);
+ }
+
+ @Test
+ public void testExecutePromptYes() throws Exception {
+ setUpTwoDatabases();
+ mockPromptWith("y");
+
+ uut.execute();
+
+ Mockito.verify(stmtMock).execute("DROP DATABASE `mydb` CASCADE");
+ Mockito.verify(stmtMock).execute(String.format("DROP TABLE `%s`.`table1`",
Warehouse.DEFAULT_DATABASE_NAME));
+ Mockito.verify(stmtMock).execute(String.format("DROP TABLE `%s`.`table2`",
Warehouse.DEFAULT_DATABASE_NAME));
+ Mockito.verify(stmtMock, times(3)).execute(anyString());
+ }
+
+ @Test
+ public void testExecutePromptNo() throws Exception {
+ setUpTwoDatabases();
+ mockPromptWith("n");
+
+ uut.execute();
+
+ Mockito.verify(stmtMock, times(0)).execute(anyString());
+ }
+
+ @Test
+ public void testExecuteDryRun() throws Exception {
+ setUpTwoDatabases();
+ when(uut.schemaTool.isDryRun()).thenReturn(true);
+
+ uut.execute();
+
+ Mockito.verify(stmtMock, times(0)).execute(anyString());
+ }
+
+ @Test
+ public void testExecuteWithYes() throws Exception {
+ setUpTwoDatabases();
+ uut.yes = true;
+
+ uut.execute();
+
+ Mockito.verify(stmtMock, times(3)).execute(anyString());
+ }
+}
