This is an automated email from the ASF dual-hosted git repository.
granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 0df93bc [hms] Allow the table type to change but not the
synchronization of a table
0df93bc is described below
commit 0df93bcd8923029eb479e70d7253237c81309dd2
Author: Grant Henke <[email protected]>
AuthorDate: Fri May 14 10:44:00 2021 -0500
[hms] Allow the table type to change but not the synchronization of a table
When upgrading some verions of the Hive Metastore it migrates Kudu tables
from managed tables to external tables with the purge property set. Both
of these states are considered “synchronized” tables by the HMS sync
and therefore this change sholuld be allowed as it won’t created orphaned
tables.
Change-Id: I7c029da1dbfef9812b0fe7389a768772605acc45
Reviewed-on: http://gerrit.cloudera.org:8080/17445
Reviewed-by: Attila Bukor <[email protected]>
Tested-by: Grant Henke <[email protected]>
---
.../kudu/hive/metastore/KuduMetastorePlugin.java | 8 ++--
.../hive/metastore/TestKuduMetastorePlugin.java | 48 ++++++++++++++--------
2 files changed, 34 insertions(+), 22 deletions(-)
diff --git
a/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
b/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
index d304146..878a362 100644
---
a/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
+++
b/java/kudu-hive/src/main/java/org/apache/kudu/hive/metastore/KuduMetastorePlugin.java
@@ -232,17 +232,15 @@ public class KuduMetastorePlugin extends
MetaStoreEventListener {
}
// Prevent altering the table type (managed/external) of Kudu tables (or
via
- // altering table properties 'EXTERNAL' or `external.table.purge`).
- // This can cause orphaned tables.
+ // altering table properties 'EXTERNAL' or `external.table.purge`) in a way
+ // that changes if a table is synchronized. This can cause orphaned tables.
// Note: This doesn't prevent altering the table type for legacy tables
// because they should continue to work as they always have primarily for
// migration purposes.
// The Kudu master is allowed to make these changes if necessary as it is
a trusted user.
if (isKuduTable(oldTable) &&
!isKuduMasterAction(tableEvent) &&
- (!Objects.equals(oldTable.getTableType(), newTable.getTableType()) ||
- isExternalTable(oldTable) != isExternalTable(newTable) ||
- isPurgeTable(oldTable) != isPurgeTable(newTable))) {
+ isSynchronizedTable(oldTable) != isSynchronizedTable(newTable) ) {
throw new MetaException("Kudu table type may not be altered");
}
diff --git
a/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
b/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
index a2b5b70..f35b025 100644
---
a/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
+++
b/java/kudu-hive/src/test/java/org/apache/kudu/hive/metastore/TestKuduMetastorePlugin.java
@@ -297,22 +297,6 @@ public class TestKuduMetastorePlugin {
"Kudu table entry must contain a Kudu storage handler
property"));
}
- // Try to alter the Kudu table to a different type.
- try {
- Table alteredTable = table.deepCopy();
- alteredTable.setTableType(TableType.MANAGED_TABLE.toString());
- // Also change the location to avoid MetastoreDefaultTransformer
validation
- // that exists in some Hive versions.
- alteredTable.getSd().setLocation(String.format("%s/%s/%s",
- clientConf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname),
- table.getDbName(), table.getTableName()));
- client.alter_table(table.getDbName(), table.getTableName(),
alteredTable);
- fail();
- } catch (TException e) {
- assertTrue(e.getMessage(),
- e.getMessage().contains("Kudu table type may not be
altered"));
- }
-
// Alter the Kudu table to a different type by setting the external
property fails.
try {
Table alteredTable = table.deepCopy();
@@ -354,7 +338,7 @@ public class TestKuduMetastorePlugin {
alteredTable.getSd().setLocation(String.format("%s/%s/%s",
clientConf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname),
table.getDbName(), table.getTableName()));
- alteredTable.putToParameters(KuduMetastorePlugin.EXTERNAL_PURGE_KEY,
"TRUE");
+ alteredTable.putToParameters(KuduMetastorePlugin.EXTERNAL_PURGE_KEY,
"FALSE");
client.alter_table(table.getDbName(), table.getTableName(),
alteredTable);
fail();
} catch (TException e) {
@@ -377,6 +361,36 @@ public class TestKuduMetastorePlugin {
alteredTable, masterContext());
}
+ // Altering the table type in a what that maintains sync works.
+ // In this case an external purge table is the same as a managed table.
+ {
+ Table alteredTable = table.deepCopy();
+ alteredTable.setTableType(TableType.MANAGED_TABLE.toString());
+ alteredTable.putToParameters(KuduMetastorePlugin.EXTERNAL_TABLE_KEY,
"FALSE");
+ alteredTable.putToParameters(KuduMetastorePlugin.EXTERNAL_PURGE_KEY,
"FALSE");
+ // Also change the location to avoid MetastoreDefaultTransformer
validation
+ // that exists in some Hive versions.
+ alteredTable.getSd().setLocation(String.format("%s/%s/%s",
+ clientConf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname),
+ table.getDbName(), table.getTableName()));
+ client.alter_table(table.getDbName(), table.getTableName(),
alteredTable);
+ }
+
+ // Altering back the table type in a what that maintains sync works.
+ // In this case a managed table is the same as an external purge table.
+ {
+ Table alteredTable = table.deepCopy();
+ alteredTable.setTableType(TableType.EXTERNAL_TABLE.toString());
+ alteredTable.putToParameters(KuduMetastorePlugin.EXTERNAL_TABLE_KEY,
"TRUE");
+ alteredTable.putToParameters(KuduMetastorePlugin.EXTERNAL_PURGE_KEY,
"TRUE");
+ // Also change the location to avoid MetastoreDefaultTransformer
validation
+ // that exists in some Hive versions.
+ alteredTable.getSd().setLocation(String.format("%s/%s/%s",
+ clientConf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname),
+ table.getDbName(), table.getTableName()));
+ client.alter_table(table.getDbName(), table.getTableName(),
alteredTable);
+ }
+
// Check that adding a column fails.
table.getSd().addToCols(new FieldSchema("b", "int", ""));
// Also change the location to avoid MetastoreDefaultTransformer
validation
