This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d519c9c HIVE-21500: Disable conversion of managed table to external
and vice versa at source via alter table (Sankar Hariappan, reviewed by
Ashutosh Bapat, Anishek Agarwal)
d519c9c is described below
commit d519c9c4cba4a1f8dae5ffc2bf685464f1fe9a24
Author: Sankar Hariappan <[email protected]>
AuthorDate: Tue Apr 16 15:43:01 2019 +0530
HIVE-21500: Disable conversion of managed table to external and vice versa
at source via alter table (Sankar Hariappan, reviewed by Ashutosh Bapat,
Anishek Agarwal)
Signed-off-by: Sankar Hariappan <[email protected]>
---
.../parse/TestReplScenariosWithStrictManaged.java | 71 ++++++++++++++++++++++
.../TestReplicationScenariosExternalTables.java | 54 ----------------
.../parse/TestReplicationWithTableMigration.java | 28 +++++++++
.../hadoop/hive/metastore/HiveAlterHandler.java | 20 ++++++
.../metastore/TransactionalValidationListener.java | 6 ++
.../metastore/utils/HiveStrictManagedUtils.java | 4 ++
6 files changed, 129 insertions(+), 54 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplScenariosWithStrictManaged.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplScenariosWithStrictManaged.java
new file mode 100644
index 0000000..db042e2
--- /dev/null
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplScenariosWithStrictManaged.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * TestReplScenariosWithStrictManaged - Test all replication scenarios with
strict managed enabled
+ * at source and target.
+ */
+public class TestReplScenariosWithStrictManaged extends
BaseReplicationAcrossInstances {
+
+ @BeforeClass
+ public static void classLevelSetup() throws Exception {
+ Map<String, String> overrides = new HashMap<>();
+ overrides.put(HiveConf.ConfVars.HIVE_STRICT_MANAGED_TABLES.varname,
"true");
+ overrides.put(MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID.getHiveName(),
"true");
+ overrides.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "true");
+ overrides.put(HiveConf.ConfVars.HIVE_TXN_MANAGER.varname,
"org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
+
+ internalBeforeClassSetup(overrides,
TestReplScenariosWithStrictManaged.class);
+ }
+
+ @Test
+ public void dynamicallyConvertManagedToExternalTable() throws Throwable {
+ // All tables are automatically converted to ACID tables when strict
managed is enabled.
+ // Also, it is not possible to convert ACID table to external table.
+ primary.run("use " + primaryDbName)
+ .run("create table t1 (id int) stored as orc")
+ .run("insert into table t1 values (1)")
+ .run("create table t2 (id int) partitioned by (key int) stored as
orc")
+ .run("insert into table t2 partition(key=10) values (1)")
+ .runFailure("alter table t1 set tblproperties('EXTERNAL'='true')")
+ .runFailure("alter table t1 set tblproperties('EXTERNAL'='true',
'TRANSACTIONAL'='false')")
+ .runFailure("alter table t2 set tblproperties('EXTERNAL'='true')");
+ }
+
+ @Test
+ public void dynamicallyConvertExternalToManagedTable() throws Throwable {
+ // With Strict managed enabled, it is not possible to convert external
table to ACID table.
+ primary.run("use " + primaryDbName)
+ .run("create external table t1 (id int) stored as orc")
+ .run("insert into table t1 values (1)")
+ .run("create external table t2 (place string) partitioned by
(country string)")
+ .run("insert into table t2 partition(country='india') values
('bangalore')")
+ .runFailure("alter table t1 set tblproperties('EXTERNAL'='false')")
+ .runFailure("alter table t1 set tblproperties('EXTERNAL'='false',
'TRANSACTIONAL'='true')")
+ .runFailure("alter table t2 set
tblproperties('EXTERNAL'='false')");
+ }
+}
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
index b8e96f2..72da2f1 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore;
import
org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection;
import
org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.CallerArguments;
-import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import
org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -674,59 +673,6 @@ public class TestReplicationScenariosExternalTables
extends BaseReplicationAcros
ErrorMsg.REPL_BOOTSTRAP_LOAD_PATH_NOT_VALID.getErrorCode());
}
- @Test
- public void dynamicallyConvertManagedToExternalTable() throws Throwable {
- List<String> dumpWithClause = Collections.singletonList(
- "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname +
"'='true'"
- );
- List<String> loadWithClause = externalTableBasePathWithClause();
-
- WarehouseInstance.Tuple tupleBootstrapManagedTable = primary.run("use " +
primaryDbName)
- .run("create table t1 (id int)")
- .run("insert into table t1 values (1)")
- .run("create table t2 (id int) partitioned by (key int)")
- .run("insert into table t2 partition(key=10) values (1)")
- .dump(primaryDbName, null, dumpWithClause);
-
- replica.load(replicatedDbName, tupleBootstrapManagedTable.dumpLocation,
loadWithClause);
-
- Hive hiveForReplica = Hive.get(replica.hiveConf);
- Table replicaTable = hiveForReplica.getTable(replicatedDbName + ".t1");
- Path oldTblLocT1 = replicaTable.getDataLocation();
-
- replicaTable = hiveForReplica.getTable(replicatedDbName + ".t2");
- Path oldTblLocT2 = replicaTable.getDataLocation();
-
- WarehouseInstance.Tuple tupleIncConvertToExternalTbl = primary.run("use "
+ primaryDbName)
- .run("alter table t1 set tblproperties('EXTERNAL'='true')")
- .run("alter table t2 set tblproperties('EXTERNAL'='true')")
- .dump(primaryDbName, tupleBootstrapManagedTable.lastReplicationId,
dumpWithClause);
-
- assertExternalFileInfo(Arrays.asList("t1", "t2"),
- new Path(tupleIncConvertToExternalTbl.dumpLocation, FILE_NAME));
- replica.load(replicatedDbName, tupleIncConvertToExternalTbl.dumpLocation,
loadWithClause)
- .run("use " + replicatedDbName)
- .run("select id from t1")
- .verifyResult("1")
- .run("select id from t2 where key=10")
- .verifyResult("1");
-
- // Check if the table type is set correctly in target.
- replicaTable = hiveForReplica.getTable(replicatedDbName + ".t1");
- assertTrue(TableType.EXTERNAL_TABLE.equals(replicaTable.getTableType()));
-
- replicaTable = hiveForReplica.getTable(replicatedDbName + ".t2");
- assertTrue(TableType.EXTERNAL_TABLE.equals(replicaTable.getTableType()));
-
- // Verify if new table location is set inside the base directory.
- assertTablePartitionLocation(primaryDbName + ".t1", replicatedDbName +
".t1");
- assertTablePartitionLocation(primaryDbName + ".t2", replicatedDbName +
".t2");
-
- // Old location should be removed and set to new location.
- assertFalse(replica.miniDFSCluster.getFileSystem().exists(oldTblLocT1));
- assertFalse(replica.miniDFSCluster.getFileSystem().exists(oldTblLocT2));
- }
-
private List<String> externalTableBasePathWithClause() throws IOException,
SemanticException {
Path externalTableLocation = new Path(REPLICA_EXTERNAL_BASE);
DistributedFileSystem fileSystem = replica.miniDFSCluster.getFileSystem();
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java
index bafcdbe..41f2b9d 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java
@@ -476,4 +476,32 @@ public class TestReplicationWithTableMigration {
replica.load(replicatedDbName, tuple.dumpLocation, withConfigs);
verifyLoadExecution(replicatedDbName, tuple.lastReplicationId);
}
+
+ @Test
+ public void dynamicallyConvertManagedToExternalTable() throws Throwable {
+ // With Strict managed disabled but Db enabled for replication, it is not
possible to convert
+ // external table to managed table.
+ primary.run("use " + primaryDbName)
+ .run("create table t1 (id int) clustered by(id) into 3 buckets
stored as orc ")
+ .run("insert into t1 values(1)")
+ .run("create table t2 partitioned by (country string) ROW FORMAT
SERDE "
+ + "'org.apache.hadoop.hive.serde2.avro.AvroSerDe' stored
as avro "
+ + "tblproperties ('avro.schema.url'='" +
avroSchemaFile.toUri().toString() + "')")
+ .run("insert into t2 partition (country='india') values
('another', 13)")
+ .runFailure("alter table t1 set tblproperties('EXTERNAL'='true')")
+ .runFailure("alter table t2 set tblproperties('EXTERNAL'='true')");
+ }
+
+ @Test
+ public void dynamicallyConvertExternalToManagedTable() throws Throwable {
+ // With Strict managed disabled but Db enabled for replication, it is not
possible to convert
+ // external table to managed table.
+ primary.run("use " + primaryDbName)
+ .run("create external table t1 (id int) stored as orc")
+ .run("insert into table t1 values (1)")
+ .run("create external table t2 (place string) partitioned by
(country string)")
+ .run("insert into table t2 partition(country='india') values
('bangalore')")
+ .runFailure("alter table t1 set tblproperties('EXTERNAL'='false')")
+ .runFailure("alter table t2 set
tblproperties('EXTERNAL'='false')");
+ }
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index ad670c9..d4aaa5c 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -169,6 +169,8 @@ public class HiveAlterHandler implements AlterHandler {
TableName.getQualified(catName, dbname, name) + " doesn't exist");
}
+ checkTableTypeConversion(olddb, oldt, newt);
+
if (oldt.getPartitionKeysSize() != 0) {
isPartitionedTable = true;
}
@@ -803,6 +805,24 @@ public class HiveAlterHandler implements AlterHandler {
return oldParts;
}
+ private void checkTableTypeConversion(Database db, Table oldTbl, Table
newTbl)
+ throws InvalidOperationException {
+ // If the given DB is enabled for replication and strict managed is false,
then table type cannot be changed.
+ // This is to avoid migration scenarios which causes Managed ACID table to
be converted to external at replica.
+ // As ACID tables cannot be converted to external table and vice versa, we
need to restrict this conversion at
+ // primary as well.
+ // Currently, table type conversion is allowed only between managed and
external table types.
+ // But, to be future proof, any table type conversion is restricted on a
replication enabled DB.
+ if
(!conf.getBoolean(MetastoreConf.ConfVars.STRICT_MANAGED_TABLES.getHiveName(),
false)
+ && !oldTbl.getTableType().equalsIgnoreCase(newTbl.getTableType())
+ && ReplChangeManager.isSourceOfReplication(db)) {
+ throw new InvalidOperationException("Table type cannot be changed from "
+ oldTbl.getTableType()
+ + " to " + newTbl.getTableType() + " for the table " +
+ TableName.getQualified(oldTbl.getCatName(), oldTbl.getDbName(),
oldTbl.getTableName())
+ + " as it is enabled for replication.");
+ }
+ }
+
private boolean checkPartialPartKeysEqual(List<FieldSchema> oldPartKeys,
List<FieldSchema> newPartKeys) {
//return true if both are null, or false if one is null and the other isn't
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index 004acf8..afa6e4c 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -146,6 +146,12 @@ public final class TransactionalValidationListener extends
MetaStorePreEventList
}
if (transactionalValuePresent) {
+ if (oldTable.getTableType().equals(TableType.MANAGED_TABLE.toString())
+ &&
newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
+ throw new MetaException(Warehouse.getQualifiedName(newTable) +
+ " cannot be converted to external table as it is transactional
table.");
+ }
+
//normalize prop name
parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL,
transactionalValue);
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java
index 7213f8e..4610cdd 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java
@@ -59,6 +59,10 @@ public class HiveStrictManagedUtils {
if (isAvroTableWithExternalSchema(table)) {
return createValidationError(table, "Managed Avro table has
externally defined schema.");
}
+ } else if (tableType == TableType.EXTERNAL_TABLE) {
+ if (MetaStoreServerUtils.isTransactionalTable(table.getParameters())) {
+ return createValidationError(table, "Table is marked as a external
table but it is transactional.");
+ }
}
}
