Repository: hive Updated Branches: refs/heads/master 4b7f88ad1 -> 091186484
http://git-wip-us.apache.org/repos/asf/hive/blob/09118648/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java index 33cf542..d9fdd29 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.metastore.events.PreEventContext; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.utils.HiveStrictManagedUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,12 +79,14 @@ public final class TransactionalValidationListener extends MetaStorePreEventList private void handle(PreAlterTableEvent context) throws MetaException { if (supportedCatalogs.contains(getTableCatalog(context.getNewTable()))) { handleAlterTableTransactionalProp(context); + HiveStrictManagedUtils.validateStrictManagedTableWithThrow(getConf(), context.getNewTable()); } } private void handle(PreCreateTableEvent context) throws MetaException { if (supportedCatalogs.contains(getTableCatalog(context.getTable()))) { handleCreateTableTransactionalProp(context); + HiveStrictManagedUtils.validateStrictManagedTableWithThrow(getConf(), context.getTable()); } } @@ -329,7 +332,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList } } - if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) { + if (MetaStoreUtils.isExternalTable(newTable)) { throw new MetaException(Warehouse.getQualifiedName(newTable) + " cannot be declared transactional because it's an external table"); } http://git-wip-us.apache.org/repos/asf/hive/blob/09118648/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 54d8830..f3b909c 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -748,6 +748,9 @@ public class MetastoreConf { "The class to use to read schemas from storage. It must implement " + "org.apache.hadoop.hive.metastore.StorageSchemaReader"), STORE_MANAGER_TYPE("datanucleus.storeManagerType", "datanucleus.storeManagerType", "rdbms", "metadata store type"), + STRICT_MANAGED_TABLES("metastore.strict.managed.tables", "hive.strict.managed.tables", false, + "Whether strict managed tables mode is enabled. With this mode enabled, " + + "only transactional tables (both full and insert-only) are allowed to be created as managed tables"), SUPPORT_SPECICAL_CHARACTERS_IN_TABLE_NAMES("metastore.support.special.characters.tablename", "hive.support.special.characters.tablename", true, "This flag should be set to true to enable support for special characters in table names.\n" http://git-wip-us.apache.org/repos/asf/hive/blob/09118648/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java new file mode 100644 index 0000000..fac3c22 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.utils; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.TableType; + +public class HiveStrictManagedUtils { + + public static void validateStrictManagedTableWithThrow(Configuration conf, Table table) + throws MetaException { + String reason = validateStrictManagedTable(conf, table); + if (reason != null) { + throw new MetaException(reason); + } + } + + /** + * Checks if the table is valid based on the rules for strict managed tables. + * @param conf + * @param table + * @return Null if the table is valid, otherwise a string message indicating why the table is invalid. + */ + public static String validateStrictManagedTable(Configuration conf, + Table table) { + if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STRICT_MANAGED_TABLES)) { + if (table.isTemporary()) { + // temp tables exempted from checks. + return null; + } + + TableType tableType = TableType.valueOf(table.getTableType()); + if (tableType == TableType.MANAGED_TABLE) { + if (!MetaStoreUtils.isTransactionalTable(table.getParameters())) { + return createValidationError(table, "Table is marked as a managed table but is not transactional."); + } + if (MetaStoreUtils.isNonNativeTable(table)) { + return createValidationError(table, "Table is marked as a managed table but is non-native."); + } + if (isAvroTableWithExternalSchema(table)) { + return createValidationError(table, "Managed Avro table has externally defined schema."); + } + } + } + + // Table is valid + return null; + } + + private static final String AVRO_SERDE_CLASSNAME = "org.apache.hadoop.hive.serde2.avro.AvroSerDe"; + private static final String AVRO_SCHEMA_URL_PROPERTY = "avro.schema.url"; + + public static boolean isAvroTableWithExternalSchema(Table table) { + if (table.getSd().getSerdeInfo().getSerializationLib().equals(AVRO_SERDE_CLASSNAME)) { + String schemaUrl = table.getParameters().get(AVRO_SCHEMA_URL_PROPERTY); + if (schemaUrl != null && !schemaUrl.isEmpty()) { + return true; + } + } + return false; + } + + public static boolean isListBucketedTable(Table table) { + return table.getSd().isStoredAsSubDirectories(); + } + + private static String createValidationError(Table table, String message) { + StringBuilder sb = new StringBuilder(); + sb.append("Table "); + sb.append(table.getDbName()); + sb.append("."); + sb.append(table.getTableName()); + sb.append(" failed strict managed table checks due to the following reason: "); + sb.append(message); + return sb.toString(); + } +}
