This is an automated email from the ASF dual-hosted git repository.
prasanthj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new acec83f HIVE-21254: Pre-upgrade tool should handle exceptions and
skip db/tables (Prasanth Jayachandran reviewed by Jason Dere, Eugene Koifman,
Ashutosh Chauhan)
acec83f is described below
commit acec83f3b7718250e6317ec5890a5ad7a014f10e
Author: Prasanth Jayachandran <prasan...@apache.org>
AuthorDate: Tue Feb 19 23:06:21 2019 -0800
HIVE-21254: Pre-upgrade tool should handle exceptions and skip db/tables
(Prasanth Jayachandran reviewed by Jason Dere, Eugene Koifman, Ashutosh Chauhan)
---
.../hadoop/hive/upgrade/acid/PreUpgradeTool.java | 93 ++++++++---
.../hive/upgrade/acid/TestPreUpgradeTool.java | 172 ++++++++++++++++-----
2 files changed, 207 insertions(+), 58 deletions(-)
diff --git
a/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java
b/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java
index 04782a6..0e3e3e2 100644
---
a/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java
+++
b/upgrade-acid/pre-upgrade/src/main/java/org/apache/hadoop/hive/upgrade/acid/PreUpgradeTool.java
@@ -25,6 +25,7 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -54,6 +55,7 @@ import org.apache.hadoop.hive.ql.io.orc.OrcFile;
import org.apache.hadoop.hive.ql.io.orc.Reader;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.common.util.HiveVersionInfo;
import org.apache.thrift.TException;
@@ -223,39 +225,74 @@ public class PreUpgradeTool {
boolean isAcidEnabled = isAcidEnabled(conf);
IMetaStoreClient hms = getHMS(conf);
LOG.debug("Looking for databases");
- List<String> databases = hms.getAllDatabases();//TException
- LOG.debug("Found " + databases.size() + " databases to process");
+ String exceptionMsg = null;
+ List<String> databases;
List<String> compactions = new ArrayList<>();
final CompactionMetaInfo compactionMetaInfo = new CompactionMetaInfo();
ValidTxnList txns = null;
Hive db = null;
- if(execute) {
- db = Hive.get(conf);
- }
+ try {
+ databases = hms.getAllDatabases();//TException
+ LOG.debug("Found " + databases.size() + " databases to process");
+ if (execute) {
+ db = Hive.get(conf);
+ }
- for(String dbName : databases) {
- List<String> tables = hms.getAllTables(dbName);
- LOG.debug("found " + tables.size() + " tables in " + dbName);
- for(String tableName : tables) {
- Table t = hms.getTable(dbName, tableName);
- LOG.debug("processing table " + Warehouse.getQualifiedName(t));
- if(isAcidEnabled) {
- //if acid is off, there can't be any acid tables - nothing to compact
- if(txns == null) {
+ for (String dbName : databases) {
+ try {
+ List<String> tables = hms.getAllTables(dbName);
+ LOG.debug("found " + tables.size() + " tables in " + dbName);
+ for (String tableName : tables) {
+ try {
+ Table t = hms.getTable(dbName, tableName);
+ LOG.debug("processing table " + Warehouse.getQualifiedName(t));
+ if (isAcidEnabled) {
+ //if acid is off, there can't be any acid tables - nothing to
compact
+ if (txns == null) {
/*
This API changed from 2.x to 3.0. so this won't even compile with
3.0
but it doesn't need to since we only run this preUpgrade
*/
- TxnStore txnHandler = TxnUtils.getTxnStore(conf);
- txns =
TxnUtils.createValidCompactTxnList(txnHandler.getOpenTxnsInfo());
+ TxnStore txnHandler = TxnUtils.getTxnStore(conf);
+ txns =
TxnUtils.createValidCompactTxnList(txnHandler.getOpenTxnsInfo());
+ }
+ List<String> compactionCommands =
+ getCompactionCommands(t, conf, hms, compactionMetaInfo,
execute, db, txns);
+ compactions.addAll(compactionCommands);
+ }
+ /*todo: handle renaming files somewhere*/
+ } catch (Exception e) {
+ if (isAccessControlException(e)) {
+ // this could be external table with 0 permission for hive user
+ exceptionMsg = "Unable to access " + dbName + "." + tableName
+ ". Pre-upgrade tool requires read-access " +
+ "to databases and tables to determine if a table has to be
compacted. " +
+ "Set " +
HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_AUTH_READS.varname + " config to
" +
+ "false to allow read-access to databases and tables and
retry the pre-upgrade tool again..";
+ }
+ throw e;
+ }
}
- List<String> compactionCommands =
- getCompactionCommands(t, conf, hms, compactionMetaInfo, execute,
db, txns);
- compactions.addAll(compactionCommands);
+ } catch (Exception e) {
+ if (exceptionMsg == null && isAccessControlException(e)) {
+ // we may not have access to read all tables from this db
+ exceptionMsg = "Unable to access " + dbName + ". Pre-upgrade tool
requires read-access " +
+ "to databases and tables to determine if a table has to be
compacted. " +
+ "Set " +
HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_AUTH_READS.varname + " config to
" +
+ "false to allow read-access to databases and tables and retry
the pre-upgrade tool again..";
+ }
+ throw e;
}
- /*todo: handle renaming files somewhere*/
}
+ } catch (Exception e) {
+ if (exceptionMsg == null && isAccessControlException(e)) {
+ exceptionMsg = "Unable to get databases. Pre-upgrade tool requires
read-access " +
+ "to databases and tables to determine if a table has to be
compacted. " +
+ "Set " +
HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_AUTH_READS.varname + " config to
" +
+ "false to allow read-access to databases and tables and retry the
pre-upgrade tool again..";
+ }
+ throw new HiveException(exceptionMsg, e);
}
+
makeCompactionScript(compactions, scriptLocation, compactionMetaInfo);
if(execute) {
@@ -306,6 +343,22 @@ public class PreUpgradeTool {
}
}
+ private boolean isAccessControlException(final Exception e) {
+ // hadoop security AccessControlException
+ if ((e instanceof MetaException && e.getCause() instanceof
AccessControlException) ||
+ ExceptionUtils.getRootCause(e) instanceof AccessControlException) {
+ return true;
+ }
+
+ // java security AccessControlException
+ if ((e instanceof MetaException && e.getCause() instanceof
java.security.AccessControlException) ||
+ ExceptionUtils.getRootCause(e) instanceof
java.security.AccessControlException) {
+ return true;
+ }
+
+ // metastore in some cases sets the AccessControlException as message
instead of wrapping the exception
+ return e instanceof MetaException &&
e.getMessage().startsWith("java.security.AccessControlException: Permission
denied");
+ }
/**
* Generates a set compaction commands to run on pre Hive 3 cluster
diff --git
a/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java
b/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java
index fe4b08b..90230d5d 100644
---
a/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java
+++
b/upgrade-acid/pre-upgrade/src/test/java/org/apache/hadoop/hive/upgrade/acid/TestPreUpgradeTool.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -46,8 +47,13 @@ import org.junit.Test;
import org.junit.rules.TestName;
import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
public class TestPreUpgradeTool {
@@ -72,41 +78,45 @@ public class TestPreUpgradeTool {
runStatementOnDriver("drop table if exists TFlat");
runStatementOnDriver("drop table if exists TFlatText");
- runStatementOnDriver("create table TAcid (a int, b int) clustered by (b)
into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
- runStatementOnDriver("create table TAcidPart (a int, b int) partitioned by
(p tinyint) clustered by (b) into 2 buckets stored" +
- " as orc TBLPROPERTIES ('transactional'='true')");
- //on 2.x these are guaranteed to not be acid
- runStatementOnDriver("create table TFlat (a int, b int) stored as orc
tblproperties('transactional'='false')");
- runStatementOnDriver("create table TFlatText (a int, b int) stored as
textfile tblproperties('transactional'='false')");
+ try {
+ runStatementOnDriver(
+ "create table TAcid (a int, b int) clustered by (b) into 2 buckets
stored as orc TBLPROPERTIES ('transactional'='true')");
+ runStatementOnDriver(
+ "create table TAcidPart (a int, b int) partitioned by (p tinyint)
clustered by (b) into 2 buckets stored" +
+ " as orc TBLPROPERTIES ('transactional'='true')");
+ //on 2.x these are guaranteed to not be acid
+ runStatementOnDriver("create table TFlat (a int, b int) stored as orc
tblproperties('transactional'='false')");
+ runStatementOnDriver(
+ "create table TFlatText (a int, b int) stored as textfile
tblproperties('transactional'='false')");
- //this needs major compaction
- runStatementOnDriver("insert into TAcid" + makeValuesClause(data));
- runStatementOnDriver("update TAcid set a = 1 where b = 2");
+ //this needs major compaction
+ runStatementOnDriver("insert into TAcid" + makeValuesClause(data));
+ runStatementOnDriver("update TAcid set a = 1 where b = 2");
- //this table needs to be converted to CRUD Acid
- runStatementOnDriver("insert into TFlat" + makeValuesClause(data));
+ //this table needs to be converted to CRUD Acid
+ runStatementOnDriver("insert into TFlat" + makeValuesClause(data));
- //this table needs to be converted to MM
- runStatementOnDriver("insert into TFlatText" + makeValuesClause(data));
+ //this table needs to be converted to MM
+ runStatementOnDriver("insert into TFlatText" + makeValuesClause(data));
- //p=10 needs major compaction
- runStatementOnDriver("insert into TAcidPart partition(p)" +
makeValuesClause(dataPart));
- runStatementOnDriver("update TAcidPart set a = 1 where b = 2 and p = 10");
+ //p=10 needs major compaction
+ runStatementOnDriver("insert into TAcidPart partition(p)" +
makeValuesClause(dataPart));
+ runStatementOnDriver("update TAcidPart set a = 1 where b = 2 and p =
10");
- //todo: add partitioned table that needs conversion to MM/Acid
+ //todo: add partitioned table that needs conversion to MM/Acid
- //todo: rename files case
- String[] args = {"-location", getTestDataDir(), "-execute"};
- PreUpgradeTool.callback = new PreUpgradeTool.Callback() {
- @Override
- void onWaitForCompaction() throws MetaException {
- runWorker(hiveConf);
- }
- };
- PreUpgradeTool.pollIntervalMs = 1;
- PreUpgradeTool.hiveConf = hiveConf;
- PreUpgradeTool.main(args);
+ //todo: rename files case
+ String[] args = {"-location", getTestDataDir(), "-execute"};
+ PreUpgradeTool.callback = new PreUpgradeTool.Callback() {
+ @Override
+ void onWaitForCompaction() throws MetaException {
+ runWorker(hiveConf);
+ }
+ };
+ PreUpgradeTool.pollIntervalMs = 1;
+ PreUpgradeTool.hiveConf = hiveConf;
+ PreUpgradeTool.main(args);
/*
todo: parse
target/tmp/org.apache.hadoop.hive.upgrade.acid.TestPreUpgradeTool-1527286256834/compacts_1527286277624.sql
@@ -115,19 +125,99 @@ public class TestPreUpgradeTool {
ALTER TABLE default.tacidpart PARTITION(p=10Y) COMPACT 'major';
* */
- TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
+ TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
+
+ ShowCompactResponse resp = txnHandler.showCompact(new
ShowCompactRequest());
+ Assert.assertEquals(2, resp.getCompactsSize());
+ for (ShowCompactResponseElement e : resp.getCompacts()) {
+ Assert.assertEquals(e.toString(), TxnStore.CLEANING_RESPONSE,
e.getState());
+ }
- ShowCompactResponse resp = txnHandler.showCompact(new
ShowCompactRequest());
- Assert.assertEquals(2, resp.getCompactsSize());
- for(ShowCompactResponseElement e : resp.getCompacts()) {
- Assert.assertEquals(e.toString(), TxnStore.CLEANING_RESPONSE,
e.getState());
+ String[] args2 = {"-location", getTestDataDir()};
+ PreUpgradeTool.main(args2);
+ /*
+ * todo: parse compacts script - make sure there is nothing in it
+ * */
+ } finally {
+ runStatementOnDriver("drop table if exists TAcid");
+ runStatementOnDriver("drop table if exists TAcidPart");
+ runStatementOnDriver("drop table if exists TFlat");
+ runStatementOnDriver("drop table if exists TFlatText");
}
+ }
- String[] args2 = {"-location", getTestDataDir()};
- PreUpgradeTool.main(args2);
- /*
- * todo: parse compacts script - make sure there is nothing in it
- * */
+ @Test
+ public void testUpgradeExternalTableNoReadPermissionForDatabase() throws
Exception {
+ int[][] data = {{1,2}, {3, 4}, {5, 6}};
+
+ runStatementOnDriver("drop database if exists test cascade");
+ runStatementOnDriver("drop table if exists TExternal");
+
+ runStatementOnDriver("create database test");
+ runStatementOnDriver("create table test.TExternal (a int, b int) stored as
orc tblproperties" +
+ "('transactional'='false')");
+
+ //this needs major compaction
+ runStatementOnDriver("insert into test.TExternal" +
makeValuesClause(data));
+
+ String dbDir = getWarehouseDir() + "/test.db";
+ File dbPath = new File(dbDir);
+ try {
+ Set<PosixFilePermission> perms =
PosixFilePermissions.fromString("-w-------");
+ Files.setPosixFilePermissions(dbPath.toPath(), perms);
+ String[] args = {"-location", getTestDataDir(), "-execute"};
+ PreUpgradeTool.pollIntervalMs = 1;
+ PreUpgradeTool.hiveConf = hiveConf;
+ Exception expected = null;
+ try {
+ PreUpgradeTool.main(args);
+ } catch (Exception e) {
+ expected = e;
+ }
+
+ Assert.assertNotNull(expected);
+ Assert.assertTrue(expected instanceof HiveException);
+ Assert.assertTrue(expected.getMessage().contains("Pre-upgrade tool
requires " +
+ "read-access to databases and tables to determine if a table has to be
compacted."));
+ } finally {
+ Set<PosixFilePermission> perms =
PosixFilePermissions.fromString("rwxrw----");
+ Files.setPosixFilePermissions(dbPath.toPath(), perms);
+ }
+ }
+
+ @Test
+ public void testUpgradeExternalTableNoReadPermissionForTable() throws
Exception {
+ int[][] data = {{1,2}, {3, 4}, {5, 6}};
+ runStatementOnDriver("drop table if exists TExternal");
+
+ runStatementOnDriver("create table TExternal (a int, b int) stored as orc
tblproperties('transactional'='false')");
+
+ //this needs major compaction
+ runStatementOnDriver("insert into TExternal" + makeValuesClause(data));
+
+ String tableDir = getWarehouseDir() + "/texternal";
+ File tablePath = new File(tableDir);
+ try {
+ Set<PosixFilePermission> perms =
PosixFilePermissions.fromString("-w-------");
+ Files.setPosixFilePermissions(tablePath.toPath(), perms);
+ String[] args = {"-location", getTestDataDir(), "-execute"};
+ PreUpgradeTool.pollIntervalMs = 1;
+ PreUpgradeTool.hiveConf = hiveConf;
+ Exception expected = null;
+ try {
+ PreUpgradeTool.main(args);
+ } catch (Exception e) {
+ expected = e;
+ }
+
+ Assert.assertNotNull(expected);
+ Assert.assertTrue(expected instanceof HiveException);
+ Assert.assertTrue(expected.getMessage().contains("Pre-upgrade tool
requires" +
+ " read-access to databases and tables to determine if a table has to
be compacted."));
+ } finally {
+ Set<PosixFilePermission> perms =
PosixFilePermissions.fromString("rwxrw----");
+ Files.setPosixFilePermissions(tablePath.toPath(), perms);
+ }
}
private static void runWorker(HiveConf hiveConf) throws MetaException {
@@ -203,6 +293,12 @@ ALTER TABLE default.tacidpart PARTITION(p=10Y) COMPACT
'major';
hiveConf
.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
"org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
+ hiveConf
+ .setVar(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS,
+
"org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener");
+ hiveConf
+ .setVar(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER,
+
"org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider");
hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK,
true);
hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false);
TxnDbUtil.setConfValues(hiveConf);
