HIVE-19753: Strict managed tables mode in Hive (Jason Dere, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c96e9edd Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c96e9edd Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c96e9edd Branch: refs/heads/branch-3 Commit: c96e9eddea3b73f7bbb7855cbda614a0fa16db71 Parents: 5c98a91 Author: Jason Dere <[email protected]> Authored: Sun Jun 24 17:45:23 2018 -0700 Committer: Jason Dere <[email protected]> Committed: Sun Jun 24 17:45:23 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 4 - .../test/resources/testconfiguration.properties | 3 + .../upgrade/hive/hive-schema-3.0.0.hive.sql | 88 +- .../upgrade/hive/hive-schema-3.1.0.hive.sql | 88 +- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 16 + .../ql/util/HiveStrictManagedMigration.java | 1 + .../hive/ql/util/HiveStrictManagedUtils.java | 115 - .../clientnegative/strict_managed_tables1.q | 15 + .../clientnegative/strict_managed_tables2.q | 7 + .../clientnegative/strict_managed_tables3.q | 7 + .../clientnegative/strict_managed_tables4.q | 16 + .../clientnegative/strict_managed_tables5.q | 12 + .../clientpositive/strict_managed_tables1.q | 40 + .../clientpositive/strict_managed_tables2.q | 46 + .../strict_managed_tables_sysdb.q | 48 + .../clientnegative/strict_managed_tables1.q.out | 29 + .../clientnegative/strict_managed_tables2.q.out | 13 + .../clientnegative/strict_managed_tables3.q.out | 13 + .../clientnegative/strict_managed_tables4.q.out | 31 + .../clientnegative/strict_managed_tables5.q.out | 19 + .../clientpositive/llap/resourceplan.q.out | 172 +- .../llap/strict_managed_tables1.q.out | 176 + .../llap/strict_managed_tables2.q.out | 248 ++ .../llap/strict_managed_tables_sysdb.q.out | 3458 ++++++++++++++++++ .../TransactionalValidationListener.java | 5 +- .../hive/metastore/conf/MetastoreConf.java | 3 + .../metastore/utils/HiveStrictManagedUtils.java | 100 + 27 files changed, 4479 insertions(+), 294 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index a383fa0..6281dc3 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2842,10 +2842,6 @@ public class HiveConf extends Configuration { " on the assumption that data changes by external applications may have negative effects" + " on these operations."), - HIVE_STRICT_MANAGED_TABLES("hive.strict.managed.tables", false, - "Whether strict managed tables mode is enabled. With this mode enabled, " + - "only transactional tables (both full and insert-only) are allowed to be created as managed tables"), - HIVE_ERROR_ON_EMPTY_PARTITION("hive.error.on.empty.partition", false, "Whether to throw an exception if dynamic partition insert generates empty results."), http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index b98e587..5e71d7c 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -695,6 +695,9 @@ minillaplocal.query.files=\ sqlmerge.q,\ stats_based_fetch_decision.q,\ stats_only_external.q,\ + strict_managed_tables_sysdb.q,\ + strict_managed_tables1.q,\ + strict_managed_tables2.q,\ subquery_in_having.q,\ subquery_notin.q,\ subquery_nested_subquery.q, \ http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql b/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql index a3ecded..71671c4 100644 --- a/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql +++ b/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql @@ -5,7 +5,7 @@ CREATE DATABASE SYS; USE SYS; -CREATE TABLE IF NOT EXISTS `BUCKETING_COLS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `BUCKETING_COLS` ( `SD_ID` bigint, `BUCKET_COL_NAME` string, `INTEGER_IDX` int, @@ -23,7 +23,7 @@ FROM \"BUCKETING_COLS\"" ); -CREATE TABLE IF NOT EXISTS `CDS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `CDS` ( `CD_ID` bigint, CONSTRAINT `SYS_PK_CDS` PRIMARY KEY (`CD_ID`) DISABLE ) @@ -37,7 +37,7 @@ FROM \"CDS\"" ); -CREATE TABLE IF NOT EXISTS `COLUMNS_V2` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `COLUMNS_V2` ( `CD_ID` bigint, `COMMENT` string, `COLUMN_NAME` string, @@ -59,7 +59,7 @@ FROM \"COLUMNS_V2\"" ); -CREATE TABLE IF NOT EXISTS `DATABASE_PARAMS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `DATABASE_PARAMS` ( `DB_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -77,7 +77,7 @@ FROM \"DATABASE_PARAMS\"" ); -CREATE TABLE IF NOT EXISTS `DBS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `DBS` ( `DB_ID` bigint, `DB_LOCATION_URI` string, `NAME` string, @@ -99,7 +99,7 @@ FROM \"DBS\"" ); -CREATE TABLE IF NOT EXISTS `DB_PRIVS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `DB_PRIVS` ( `DB_GRANT_ID` bigint, `CREATE_TIME` int, `DB_ID` bigint, @@ -131,7 +131,7 @@ FROM \"DB_PRIVS\"" ); -CREATE TABLE IF NOT EXISTS `GLOBAL_PRIVS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `GLOBAL_PRIVS` ( `USER_GRANT_ID` bigint, `CREATE_TIME` int, `GRANT_OPTION` string, @@ -161,7 +161,7 @@ FROM \"GLOBAL_PRIVS\"" ); -CREATE TABLE IF NOT EXISTS `PARTITIONS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITIONS` ( `PART_ID` bigint, `CREATE_TIME` int, `LAST_ACCESS_TIME` int, @@ -185,7 +185,7 @@ FROM \"PARTITIONS\"" ); -CREATE TABLE IF NOT EXISTS `PARTITION_KEYS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITION_KEYS` ( `TBL_ID` bigint, `PKEY_COMMENT` string, `PKEY_NAME` string, @@ -207,7 +207,7 @@ FROM \"PARTITION_KEYS\"" ); -CREATE TABLE IF NOT EXISTS `PARTITION_KEY_VALS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITION_KEY_VALS` ( `PART_ID` bigint, `PART_KEY_VAL` string, `INTEGER_IDX` int, @@ -225,7 +225,7 @@ FROM \"PARTITION_KEY_VALS\"" ); -CREATE TABLE IF NOT EXISTS `PARTITION_PARAMS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITION_PARAMS` ( `PART_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -243,7 +243,7 @@ FROM \"PARTITION_PARAMS\"" ); -CREATE TABLE IF NOT EXISTS `PART_COL_PRIVS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PART_COL_PRIVS` ( `PART_COLUMN_GRANT_ID` bigint, `COLUMN_NAME` string, `CREATE_TIME` int, @@ -277,7 +277,7 @@ FROM \"PART_COL_PRIVS\"" ); -CREATE TABLE IF NOT EXISTS `PART_PRIVS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PART_PRIVS` ( `PART_GRANT_ID` bigint, `CREATE_TIME` int, `GRANT_OPTION` int, @@ -309,7 +309,7 @@ FROM \"PART_PRIVS\"" ); -CREATE TABLE IF NOT EXISTS `ROLES` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `ROLES` ( `ROLE_ID` bigint, `CREATE_TIME` int, `OWNER_NAME` string, @@ -329,7 +329,7 @@ FROM \"ROLES\"" ); -CREATE TABLE IF NOT EXISTS `ROLE_MAP` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `ROLE_MAP` ( `ROLE_GRANT_ID` bigint, `ADD_TIME` int, `GRANT_OPTION` int, @@ -357,7 +357,7 @@ FROM \"ROLE_MAP\"" ); -CREATE TABLE IF NOT EXISTS `SDS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SDS` ( `SD_ID` bigint, `CD_ID` bigint, `INPUT_FORMAT` string, @@ -387,7 +387,7 @@ FROM \"SDS\"" ); -CREATE TABLE IF NOT EXISTS `SD_PARAMS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SD_PARAMS` ( `SD_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -405,7 +405,7 @@ FROM \"SD_PARAMS\"" ); -CREATE TABLE IF NOT EXISTS `SEQUENCE_TABLE` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SEQUENCE_TABLE` ( `SEQUENCE_NAME` string, `NEXT_VAL` bigint, CONSTRAINT `SYS_PK_SEQUENCE_TABLE` PRIMARY KEY (`SEQUENCE_NAME`) DISABLE @@ -421,7 +421,7 @@ FROM \"SEQUENCE_TABLE\"" ); -CREATE TABLE IF NOT EXISTS `SERDES` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SERDES` ( `SERDE_ID` bigint, `NAME` string, `SLIB` string, @@ -439,7 +439,7 @@ FROM \"SERDES\"" ); -CREATE TABLE IF NOT EXISTS `SERDE_PARAMS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SERDE_PARAMS` ( `SERDE_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -457,7 +457,7 @@ FROM \"SERDE_PARAMS\"" ); -CREATE TABLE IF NOT EXISTS `SKEWED_COL_NAMES` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_COL_NAMES` ( `SD_ID` bigint, `SKEWED_COL_NAME` string, `INTEGER_IDX` int, @@ -475,7 +475,7 @@ FROM \"SKEWED_COL_NAMES\"" ); -CREATE TABLE IF NOT EXISTS `SKEWED_COL_VALUE_LOC_MAP` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_COL_VALUE_LOC_MAP` ( `SD_ID` bigint, `STRING_LIST_ID_KID` bigint, `LOCATION` string, @@ -493,7 +493,7 @@ FROM \"SKEWED_COL_VALUE_LOC_MAP\"" ); -CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_STRING_LIST` ( `STRING_LIST_ID` bigint, CONSTRAINT `SYS_PK_SKEWED_STRING_LIST` PRIMARY KEY (`STRING_LIST_ID`) DISABLE ) @@ -507,7 +507,7 @@ FROM \"SKEWED_STRING_LIST\"" ); -CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST_VALUES` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_STRING_LIST_VALUES` ( `STRING_LIST_ID` bigint, `STRING_LIST_VALUE` string, `INTEGER_IDX` int, @@ -525,7 +525,7 @@ FROM \"SKEWED_STRING_LIST_VALUES\"" ); -CREATE TABLE IF NOT EXISTS `SKEWED_VALUES` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_VALUES` ( `SD_ID_OID` bigint, `STRING_LIST_ID_EID` bigint, `INTEGER_IDX` int, @@ -543,7 +543,7 @@ FROM \"SKEWED_VALUES\"" ); -CREATE TABLE IF NOT EXISTS `SORT_COLS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SORT_COLS` ( `SD_ID` bigint, `COLUMN_NAME` string, `ORDER` int, @@ -563,7 +563,7 @@ FROM \"SORT_COLS\"" ); -CREATE TABLE IF NOT EXISTS `TABLE_PARAMS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `TABLE_PARAMS` ( `TBL_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -581,7 +581,7 @@ FROM \"TABLE_PARAMS\"" ); -CREATE TABLE IF NOT EXISTS `TBLS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `TBLS` ( `TBL_ID` bigint, `CREATE_TIME` int, `DB_ID` bigint, @@ -616,7 +616,7 @@ TBLPROPERTIES ( FROM \"TBLS\"" ); -CREATE TABLE IF NOT EXISTS `MV_CREATION_METADATA` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `MV_CREATION_METADATA` ( `MV_CREATION_METADATA_ID` bigint, `DB_NAME` string, `TBL_NAME` string, @@ -635,7 +635,7 @@ TBLPROPERTIES ( FROM \"MV_CREATION_METADATA\"" ); -CREATE TABLE IF NOT EXISTS `MV_TABLES_USED` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `MV_TABLES_USED` ( `MV_CREATION_METADATA_ID` bigint, `TBL_ID` bigint ) @@ -649,7 +649,7 @@ TBLPROPERTIES ( FROM \"MV_TABLES_USED\"" ); -CREATE TABLE IF NOT EXISTS `TBL_COL_PRIVS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `TBL_COL_PRIVS` ( `TBL_COLUMN_GRANT_ID` bigint, `COLUMN_NAME` string, `CREATE_TIME` int, @@ -683,7 +683,7 @@ FROM \"TBL_COL_PRIVS\"" ); -CREATE TABLE IF NOT EXISTS `TBL_PRIVS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `TBL_PRIVS` ( `TBL_GRANT_ID` bigint, `CREATE_TIME` int, `GRANT_OPTION` int, @@ -715,7 +715,7 @@ FROM \"TBL_PRIVS\"" ); -CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `TAB_COL_STATS` ( `CS_ID` bigint, `DB_NAME` string, `TABLE_NAME` string, @@ -765,7 +765,7 @@ FROM \"TAB_COL_STATS\"" ); -CREATE TABLE IF NOT EXISTS `PART_COL_STATS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PART_COL_STATS` ( `CS_ID` bigint, `DB_NAME` string, `TABLE_NAME` string, @@ -817,7 +817,7 @@ FROM \"PART_COL_STATS\"" ); -CREATE TABLE IF NOT EXISTS `VERSION` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `VERSION` ( `VER_ID` BIGINT, `SCHEMA_VERSION` string, `VERSION_COMMENT` string, @@ -826,7 +826,7 @@ CREATE TABLE IF NOT EXISTS `VERSION` ( INSERT INTO `VERSION` VALUES (1, '3.0.0', 'Hive release version 3.0.0'); -CREATE TABLE IF NOT EXISTS `DB_VERSION` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `DB_VERSION` ( `VER_ID` BIGINT, `SCHEMA_VERSION` string, `VERSION_COMMENT` string, @@ -844,7 +844,7 @@ FROM \"VERSION\"" ); -CREATE TABLE IF NOT EXISTS `FUNCS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `FUNCS` ( `FUNC_ID` bigint, `CLASS_NAME` string, `CREATE_TIME` int, @@ -872,7 +872,7 @@ FROM \"FUNCS\"" ); --- CREATE TABLE IF NOT EXISTS `FUNC_RU` ( +-- CREATE EXTERNAL TABLE IF NOT EXISTS `FUNC_RU` ( -- `FUNC_ID` bigint, -- `RESOURCE_TYPE` int, -- `RESOURCE_URI` string, @@ -885,7 +885,7 @@ FROM -- "hive.sql.query" = "SELECT * FROM FUNCS_RU" -- ); -CREATE TABLE IF NOT EXISTS `KEY_CONSTRAINTS` +CREATE EXTERNAL TABLE IF NOT EXISTS `KEY_CONSTRAINTS` ( `CHILD_CD_ID` bigint, `CHILD_INTEGER_IDX` int, @@ -946,7 +946,7 @@ SELECT max(CASE `PARAM_KEY` WHEN 'transient_lastDdlTime' THEN `PARAM_VALUE` END) AS TRANSIENT_LAST_DDL_TIME FROM `PARTITION_PARAMS` GROUP BY `PART_ID`; -CREATE TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( `NAME` string, `STATUS` string, `QUERY_PARALLELISM` int, @@ -965,7 +965,7 @@ FROM \"WM_RESOURCEPLAN\" LEFT OUTER JOIN \"WM_POOL\" ON \"WM_RESOURCEPLAN\".\"DEFAULT_POOL_ID\" = \"WM_POOL\".\"POOL_ID\"" ); -CREATE TABLE IF NOT EXISTS `WM_TRIGGERS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_TRIGGERS` ( `RP_NAME` string, `NAME` string, `TRIGGER_EXPRESSION` string, @@ -988,7 +988,7 @@ ON t.\"RP_ID\" = r.\"RP_ID\"" ); -CREATE TABLE IF NOT EXISTS `WM_POOLS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS` ( `RP_NAME` string, `PATH` string, `ALLOC_FRACTION` double, @@ -1013,7 +1013,7 @@ ON \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\"" ); -CREATE TABLE IF NOT EXISTS `WM_POOLS_TO_TRIGGERS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS_TO_TRIGGERS` ( `RP_NAME` string, `POOL_PATH` string, `TRIGGER_NAME` string @@ -1041,7 +1041,7 @@ WHERE CAST(\"WM_TRIGGER\".\"IS_IN_UNMANAGED\" AS CHAR) IN ('1', 't') " ); -CREATE TABLE IF NOT EXISTS `WM_MAPPINGS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_MAPPINGS` ( `RP_NAME` string, `ENTITY_TYPE` string, `ENTITY_NAME` string, http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql b/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql index 8453ee3..9bfe9c2 100644 --- a/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql +++ b/metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql @@ -5,7 +5,7 @@ CREATE DATABASE SYS; USE SYS; -CREATE TABLE IF NOT EXISTS `BUCKETING_COLS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `BUCKETING_COLS` ( `SD_ID` bigint, `BUCKET_COL_NAME` string, `INTEGER_IDX` int, @@ -23,7 +23,7 @@ FROM \"BUCKETING_COLS\"" ); -CREATE TABLE IF NOT EXISTS `CDS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `CDS` ( `CD_ID` bigint, CONSTRAINT `SYS_PK_CDS` PRIMARY KEY (`CD_ID`) DISABLE ) @@ -37,7 +37,7 @@ FROM \"CDS\"" ); -CREATE TABLE IF NOT EXISTS `COLUMNS_V2` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `COLUMNS_V2` ( `CD_ID` bigint, `COMMENT` string, `COLUMN_NAME` string, @@ -59,7 +59,7 @@ FROM \"COLUMNS_V2\"" ); -CREATE TABLE IF NOT EXISTS `DATABASE_PARAMS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `DATABASE_PARAMS` ( `DB_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -77,7 +77,7 @@ FROM \"DATABASE_PARAMS\"" ); -CREATE TABLE IF NOT EXISTS `DBS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `DBS` ( `DB_ID` bigint, `DB_LOCATION_URI` string, `NAME` string, @@ -99,7 +99,7 @@ FROM \"DBS\"" ); -CREATE TABLE IF NOT EXISTS `DB_PRIVS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `DB_PRIVS` ( `DB_GRANT_ID` bigint, `CREATE_TIME` int, `DB_ID` bigint, @@ -131,7 +131,7 @@ FROM \"DB_PRIVS\"" ); -CREATE TABLE IF NOT EXISTS `GLOBAL_PRIVS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `GLOBAL_PRIVS` ( `USER_GRANT_ID` bigint, `CREATE_TIME` int, `GRANT_OPTION` string, @@ -161,7 +161,7 @@ FROM \"GLOBAL_PRIVS\"" ); -CREATE TABLE IF NOT EXISTS `PARTITIONS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITIONS` ( `PART_ID` bigint, `CREATE_TIME` int, `LAST_ACCESS_TIME` int, @@ -185,7 +185,7 @@ FROM \"PARTITIONS\"" ); -CREATE TABLE IF NOT EXISTS `PARTITION_KEYS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITION_KEYS` ( `TBL_ID` bigint, `PKEY_COMMENT` string, `PKEY_NAME` string, @@ -207,7 +207,7 @@ FROM \"PARTITION_KEYS\"" ); -CREATE TABLE IF NOT EXISTS `PARTITION_KEY_VALS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITION_KEY_VALS` ( `PART_ID` bigint, `PART_KEY_VAL` string, `INTEGER_IDX` int, @@ -225,7 +225,7 @@ FROM \"PARTITION_KEY_VALS\"" ); -CREATE TABLE IF NOT EXISTS `PARTITION_PARAMS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITION_PARAMS` ( `PART_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -243,7 +243,7 @@ FROM \"PARTITION_PARAMS\"" ); -CREATE TABLE IF NOT EXISTS `PART_COL_PRIVS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PART_COL_PRIVS` ( `PART_COLUMN_GRANT_ID` bigint, `COLUMN_NAME` string, `CREATE_TIME` int, @@ -277,7 +277,7 @@ FROM \"PART_COL_PRIVS\"" ); -CREATE TABLE IF NOT EXISTS `PART_PRIVS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PART_PRIVS` ( `PART_GRANT_ID` bigint, `CREATE_TIME` int, `GRANT_OPTION` int, @@ -309,7 +309,7 @@ FROM \"PART_PRIVS\"" ); -CREATE TABLE IF NOT EXISTS `ROLES` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `ROLES` ( `ROLE_ID` bigint, `CREATE_TIME` int, `OWNER_NAME` string, @@ -329,7 +329,7 @@ FROM \"ROLES\"" ); -CREATE TABLE IF NOT EXISTS `ROLE_MAP` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `ROLE_MAP` ( `ROLE_GRANT_ID` bigint, `ADD_TIME` int, `GRANT_OPTION` int, @@ -357,7 +357,7 @@ FROM \"ROLE_MAP\"" ); -CREATE TABLE IF NOT EXISTS `SDS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SDS` ( `SD_ID` bigint, `CD_ID` bigint, `INPUT_FORMAT` string, @@ -387,7 +387,7 @@ FROM \"SDS\"" ); -CREATE TABLE IF NOT EXISTS `SD_PARAMS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SD_PARAMS` ( `SD_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -405,7 +405,7 @@ FROM \"SD_PARAMS\"" ); -CREATE TABLE IF NOT EXISTS `SEQUENCE_TABLE` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SEQUENCE_TABLE` ( `SEQUENCE_NAME` string, `NEXT_VAL` bigint, CONSTRAINT `SYS_PK_SEQUENCE_TABLE` PRIMARY KEY (`SEQUENCE_NAME`) DISABLE @@ -421,7 +421,7 @@ FROM \"SEQUENCE_TABLE\"" ); -CREATE TABLE IF NOT EXISTS `SERDES` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SERDES` ( `SERDE_ID` bigint, `NAME` string, `SLIB` string, @@ -439,7 +439,7 @@ FROM \"SERDES\"" ); -CREATE TABLE IF NOT EXISTS `SERDE_PARAMS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SERDE_PARAMS` ( `SERDE_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -457,7 +457,7 @@ FROM \"SERDE_PARAMS\"" ); -CREATE TABLE IF NOT EXISTS `SKEWED_COL_NAMES` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_COL_NAMES` ( `SD_ID` bigint, `SKEWED_COL_NAME` string, `INTEGER_IDX` int, @@ -475,7 +475,7 @@ FROM \"SKEWED_COL_NAMES\"" ); -CREATE TABLE IF NOT EXISTS `SKEWED_COL_VALUE_LOC_MAP` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_COL_VALUE_LOC_MAP` ( `SD_ID` bigint, `STRING_LIST_ID_KID` bigint, `LOCATION` string, @@ -493,7 +493,7 @@ FROM \"SKEWED_COL_VALUE_LOC_MAP\"" ); -CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_STRING_LIST` ( `STRING_LIST_ID` bigint, CONSTRAINT `SYS_PK_SKEWED_STRING_LIST` PRIMARY KEY (`STRING_LIST_ID`) DISABLE ) @@ -507,7 +507,7 @@ FROM \"SKEWED_STRING_LIST\"" ); -CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST_VALUES` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_STRING_LIST_VALUES` ( `STRING_LIST_ID` bigint, `STRING_LIST_VALUE` string, `INTEGER_IDX` int, @@ -525,7 +525,7 @@ FROM \"SKEWED_STRING_LIST_VALUES\"" ); -CREATE TABLE IF NOT EXISTS `SKEWED_VALUES` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_VALUES` ( `SD_ID_OID` bigint, `STRING_LIST_ID_EID` bigint, `INTEGER_IDX` int, @@ -543,7 +543,7 @@ FROM \"SKEWED_VALUES\"" ); -CREATE TABLE IF NOT EXISTS `SORT_COLS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `SORT_COLS` ( `SD_ID` bigint, `COLUMN_NAME` string, `ORDER` int, @@ -563,7 +563,7 @@ FROM \"SORT_COLS\"" ); -CREATE TABLE IF NOT EXISTS `TABLE_PARAMS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `TABLE_PARAMS` ( `TBL_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -581,7 +581,7 @@ FROM \"TABLE_PARAMS\"" ); -CREATE TABLE IF NOT EXISTS `TBLS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `TBLS` ( `TBL_ID` bigint, `CREATE_TIME` int, `DB_ID` bigint, @@ -616,7 +616,7 @@ TBLPROPERTIES ( FROM \"TBLS\"" ); -CREATE TABLE IF NOT EXISTS `MV_CREATION_METADATA` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `MV_CREATION_METADATA` ( `MV_CREATION_METADATA_ID` bigint, `DB_NAME` string, `TBL_NAME` string, @@ -635,7 +635,7 @@ TBLPROPERTIES ( FROM \"MV_CREATION_METADATA\"" ); -CREATE TABLE IF NOT EXISTS `MV_TABLES_USED` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `MV_TABLES_USED` ( `MV_CREATION_METADATA_ID` bigint, `TBL_ID` bigint ) @@ -649,7 +649,7 @@ TBLPROPERTIES ( FROM \"MV_TABLES_USED\"" ); -CREATE TABLE IF NOT EXISTS `TBL_COL_PRIVS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `TBL_COL_PRIVS` ( `TBL_COLUMN_GRANT_ID` bigint, `COLUMN_NAME` string, `CREATE_TIME` int, @@ -683,7 +683,7 @@ FROM \"TBL_COL_PRIVS\"" ); -CREATE TABLE IF NOT EXISTS `TBL_PRIVS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `TBL_PRIVS` ( `TBL_GRANT_ID` bigint, `CREATE_TIME` int, `GRANT_OPTION` int, @@ -715,7 +715,7 @@ FROM \"TBL_PRIVS\"" ); -CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `TAB_COL_STATS` ( `CS_ID` bigint, `DB_NAME` string, `TABLE_NAME` string, @@ -765,7 +765,7 @@ FROM \"TAB_COL_STATS\"" ); -CREATE TABLE IF NOT EXISTS `PART_COL_STATS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `PART_COL_STATS` ( `CS_ID` bigint, `DB_NAME` string, `TABLE_NAME` string, @@ -817,7 +817,7 @@ FROM \"PART_COL_STATS\"" ); -CREATE TABLE IF NOT EXISTS `VERSION` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `VERSION` ( `VER_ID` BIGINT, `SCHEMA_VERSION` string, `VERSION_COMMENT` string, @@ -826,7 +826,7 @@ CREATE TABLE IF NOT EXISTS `VERSION` ( INSERT INTO `VERSION` VALUES (1, '3.1.0', 'Hive release version 3.1.0'); -CREATE TABLE IF NOT EXISTS `DB_VERSION` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `DB_VERSION` ( `VER_ID` BIGINT, `SCHEMA_VERSION` string, `VERSION_COMMENT` string, @@ -844,7 +844,7 @@ FROM \"VERSION\"" ); -CREATE TABLE IF NOT EXISTS `FUNCS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `FUNCS` ( `FUNC_ID` bigint, `CLASS_NAME` string, `CREATE_TIME` int, @@ -872,7 +872,7 @@ FROM \"FUNCS\"" ); --- CREATE TABLE IF NOT EXISTS `FUNC_RU` ( +-- CREATE EXTERNAL TABLE IF NOT EXISTS `FUNC_RU` ( -- `FUNC_ID` bigint, -- `RESOURCE_TYPE` int, -- `RESOURCE_URI` string, @@ -885,7 +885,7 @@ FROM -- "hive.sql.query" = "SELECT * FROM FUNCS_RU" -- ); -CREATE TABLE IF NOT EXISTS `KEY_CONSTRAINTS` +CREATE EXTERNAL TABLE IF NOT EXISTS `KEY_CONSTRAINTS` ( `CHILD_CD_ID` bigint, `CHILD_INTEGER_IDX` int, @@ -946,7 +946,7 @@ SELECT max(CASE `PARAM_KEY` WHEN 'transient_lastDdlTime' THEN `PARAM_VALUE` END) AS TRANSIENT_LAST_DDL_TIME FROM `PARTITION_PARAMS` GROUP BY `PART_ID`; -CREATE TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( `NAME` string, `STATUS` string, `QUERY_PARALLELISM` int, @@ -965,7 +965,7 @@ FROM \"WM_RESOURCEPLAN\" LEFT OUTER JOIN \"WM_POOL\" ON \"WM_RESOURCEPLAN\".\"DEFAULT_POOL_ID\" = \"WM_POOL\".\"POOL_ID\"" ); -CREATE TABLE IF NOT EXISTS `WM_TRIGGERS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_TRIGGERS` ( `RP_NAME` string, `NAME` string, `TRIGGER_EXPRESSION` string, @@ -988,7 +988,7 @@ ON t.\"RP_ID\" = r.\"RP_ID\"" ); -CREATE TABLE IF NOT EXISTS `WM_POOLS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS` ( `RP_NAME` string, `PATH` string, `ALLOC_FRACTION` double, @@ -1013,7 +1013,7 @@ ON \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\"" ); -CREATE TABLE IF NOT EXISTS `WM_POOLS_TO_TRIGGERS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS_TO_TRIGGERS` ( `RP_NAME` string, `POOL_PATH` string, `TRIGGER_NAME` string @@ -1041,7 +1041,7 @@ WHERE CAST(\"WM_TRIGGER\".\"IS_IN_UNMANAGED\" AS CHAR) IN ('1', 't') " ); -CREATE TABLE IF NOT EXISTS `WM_MAPPINGS` ( +CREATE EXTERNAL TABLE IF NOT EXISTS `WM_MAPPINGS` ( `RP_NAME` string, `ENTITY_TYPE` string, `ENTITY_NAME` string, http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index a1101f8..acca490 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -4389,6 +4389,11 @@ public class DDLTask extends Task<DDLWork> implements Serializable { if (isFromMmTable && isRemoved) { throw new HiveException("Cannot convert an ACID table to non-ACID"); } + + // Check if external table property being removed + if (removedSet.contains("EXTERNAL") && tbl.getTableType() == TableType.EXTERNAL_TABLE) { + tbl.setTableType(TableType.MANAGED_TABLE); + } } Iterator<String> keyItr = alterTbl.getProps().keySet().iterator(); while (keyItr.hasNext()) { @@ -4494,6 +4499,17 @@ public class DDLTask extends Task<DDLWork> implements Serializable { throw new HiveException("Cannot convert an ACID table to non-ACID"); } } + + // Converting to/from external table + String externalProp = alterTbl.getProps().get("EXTERNAL"); + if (externalProp != null) { + if (Boolean.parseBoolean(externalProp) && tbl.getTableType() == TableType.MANAGED_TABLE) { + tbl.setTableType(TableType.EXTERNAL_TABLE); + } else if (!Boolean.parseBoolean(externalProp) && tbl.getTableType() == TableType.EXTERNAL_TABLE) { + tbl.setTableType(TableType.MANAGED_TABLE); + } + } + tbl.getTTable().getParameters().putAll(alterTbl.getProps()); } return result; http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java index 604e4d2..0f0dc22 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.TransactionalValidationListener; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.utils.HiveStrictManagedUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.DriverFactory; import org.apache.hadoop.hive.ql.IDriver; http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedUtils.java deleted file mode 100644 index d9536eb..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedUtils.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.util; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; -import org.apache.hadoop.hive.ql.io.AcidUtils; -import org.apache.hadoop.hive.ql.io.NullRowsInputFormat; -import org.apache.hadoop.hive.ql.io.OneNullRowInputFormat; -import org.apache.hadoop.hive.ql.io.ZeroRowsInputFormat; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.serde2.avro.AvroSerDe; -import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils.AvroTableProperties; - -public class HiveStrictManagedUtils { - - private final static Set<String> EXEMPT_INPUTFORMATS = - new HashSet<String>(Arrays.asList(NullRowsInputFormat.class.getName(), - OneNullRowInputFormat.class.getName(), ZeroRowsInputFormat.class.getName())); - - - public static void validateStrictManagedTable(Configuration conf, Table table) - throws HiveException { - String reason = validateStrictManagedTable(conf, table.getTTable()); - if (reason != null) { - throw new HiveException(reason); - } - } - - /** - * Checks if the table is valid based on the rules for strict managed tables. - * @param conf - * @param table - * @return Null if the table is valid, otherwise a string message indicating why the table is invalid. - */ - public static String validateStrictManagedTable(Configuration conf, - org.apache.hadoop.hive.metastore.api.Table table) { - if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STRICT_MANAGED_TABLES)) { - if (table.isTemporary()) { - // temp tables exempted from checks. - return null; - } - - TableType tableType = TableType.valueOf(table.getTableType()); - if (tableType == TableType.MANAGED_TABLE) { - if (!AcidUtils.isTransactionalTable(table)) { - String inputFormat = null; - if (table.getSd() != null) { - inputFormat = table.getSd().getInputFormat(); - } - if (!EXEMPT_INPUTFORMATS.contains(inputFormat)) { - return createValidationError(table, "Table is marked as a managed table but is not transactional."); - } - } - if (MetaStoreUtils.isNonNativeTable(table)) { - return createValidationError(table, "Table is marked as a managed table but is non-native."); - } - if (isAvroTableWithExternalSchema(table)) { - return createValidationError(table, "Managed Avro table has externally defined schema."); - } - } - } - - // Table is valid - return null; - } - - public static boolean isAvroTableWithExternalSchema(org.apache.hadoop.hive.metastore.api.Table table) { - if (table.getSd().getSerdeInfo().getSerializationLib().equals(AvroSerDe.class.getName())) { - String schemaUrl = table.getParameters().get(AvroTableProperties.SCHEMA_URL.getPropName()); - if (schemaUrl != null && !schemaUrl.isEmpty()) { - return true; - } - } - return false; - } - - public static boolean isListBucketedTable(org.apache.hadoop.hive.metastore.api.Table table) { - return table.getSd().isStoredAsSubDirectories(); - } - - private static String createValidationError(org.apache.hadoop.hive.metastore.api.Table table, String message) { - StringBuilder sb = new StringBuilder(); - sb.append("Table "); - sb.append(table.getDbName()); - sb.append("."); - sb.append(table.getTableName()); - sb.append(" failed strict managed table checks due to the following reason: "); - sb.append(message); - return sb.toString(); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/queries/clientnegative/strict_managed_tables1.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/strict_managed_tables1.q b/ql/src/test/queries/clientnegative/strict_managed_tables1.q new file mode 100644 index 0000000..739ab49 --- /dev/null +++ b/ql/src/test/queries/clientnegative/strict_managed_tables1.q @@ -0,0 +1,15 @@ +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set metastore.strict.managed.tables=true; + +-- Managed transactional table ok +create table strict_managed_tables1_tab1 (c1 string, c2 string) stored as orc tblproperties ('transactional'='true'); + +-- Managed insert-only transactional table ok +create table strict_managed_tables1_tab2 (c1 string, c2 string) stored as textfile tblproperties ('transactional'='true', 'transactional_properties'='insert_only'); + +-- External non-transactional table ok +create external table strict_managed_tables1_tab3 (c1 string, c2 string) stored as textfile; + +-- Managed non-transactional table not ok +create table strict_managed_tables1_tab4 (c1 string, c2 string) stored as textfile; http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/queries/clientnegative/strict_managed_tables2.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/strict_managed_tables2.q b/ql/src/test/queries/clientnegative/strict_managed_tables2.q new file mode 100644 index 0000000..080c0f2 --- /dev/null +++ b/ql/src/test/queries/clientnegative/strict_managed_tables2.q @@ -0,0 +1,7 @@ +set metastore.strict.managed.tables=true; + +-- External non-transactional table ok +create external table strict_managed_tables2_tab1 (c1 string, c2 string) stored as textfile; + +-- Trying to change the table to non-external is not +alter table strict_managed_tables2_tab1 set tblproperties ('EXTERNAL'='FALSE'); http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/queries/clientnegative/strict_managed_tables3.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/strict_managed_tables3.q b/ql/src/test/queries/clientnegative/strict_managed_tables3.q new file mode 100644 index 0000000..c0e28d2 --- /dev/null +++ b/ql/src/test/queries/clientnegative/strict_managed_tables3.q @@ -0,0 +1,7 @@ +set metastore.strict.managed.tables=true; + +-- External non-transactional table ok +create external table strict_managed_tables3_tab1 (c1 string, c2 string) stored as textfile; + +-- Trying to change the table to non-external is not +alter table strict_managed_tables3_tab1 unset tblproperties ('EXTERNAL'); http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/queries/clientnegative/strict_managed_tables4.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/strict_managed_tables4.q b/ql/src/test/queries/clientnegative/strict_managed_tables4.q new file mode 100644 index 0000000..34d477a --- /dev/null +++ b/ql/src/test/queries/clientnegative/strict_managed_tables4.q @@ -0,0 +1,16 @@ + +set metastore.strict.managed.tables=true; + +dfs -rm -f ${system:test.tmp.dir}/smt6_schema.avsc; +dfs -cp ${system:hive.root}data/files/grad.avsc ${system:test.tmp.dir}/smt6_schema.avsc; + +CREATE EXTERNAL TABLE strict_managed_tables6_tab1 +STORED AS AVRO +TBLPROPERTIES ('avro.schema.url'='${system:test.tmp.dir}/smt6_schema.avsc'); + +describe strict_managed_tables6_tab1; + +-- avro table with external schema url not allowed as managed table +CREATE TABLE strict_managed_tables6_tab2 +STORED AS AVRO +TBLPROPERTIES ('avro.schema.url'='${system:test.tmp.dir}/smt6_schema.avsc'); http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/queries/clientnegative/strict_managed_tables5.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/strict_managed_tables5.q b/ql/src/test/queries/clientnegative/strict_managed_tables5.q new file mode 100644 index 0000000..c539166 --- /dev/null +++ b/ql/src/test/queries/clientnegative/strict_managed_tables5.q @@ -0,0 +1,12 @@ + +set metastore.strict.managed.tables=true; + +create external table strict_managed_tables5_tab1 (c1 string, c2 string) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'; + +-- Managed non-native table should fail +create table strict_managed_tables5_tab2 (c1 string, c2 string) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'; + http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/queries/clientpositive/strict_managed_tables1.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/strict_managed_tables1.q b/ql/src/test/queries/clientpositive/strict_managed_tables1.q new file mode 100644 index 0000000..9057e04 --- /dev/null +++ b/ql/src/test/queries/clientpositive/strict_managed_tables1.q @@ -0,0 +1,40 @@ + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +drop table if exists smt1_tab1; +drop table if exists smt1_tab2; + +create table smt1_tab1 (c1 string, c2 string) stored as textfile; +load data local inpath '../../data/files/kv1.txt' into table smt1_tab1; +select * from smt1_tab1 where c1 = '0'; + +create table smt1_tab2 (c1 string, c2 string) stored as textfile; +insert into table smt1_tab2 select * from smt1_tab1; +select * from smt1_tab2 where c1 = '0'; + +-- After this point, managed non-transactional table not valid +set metastore.strict.managed.tables=true; + +-- Setting to external table should allow table to be usable +alter table smt1_tab1 set tblproperties('EXTERNAL'='TRUE'); +select * from smt1_tab1 where c1 = '0'; + +alter table smt1_tab2 set tblproperties('EXTERNAL'='TRUE'); +select * from smt1_tab2 where c1 = '0'; + +-- Setting to managed insert-only transactional table should allow table to be usable +alter table smt1_tab1 set tblproperties('EXTERNAL'='FALSE', 'transactional'='true', 'transactional_properties'='insert_only'); +select * from smt1_tab1 where c1 = '0'; + +alter table smt1_tab2 set tblproperties('EXTERNAL'='FALSE', 'transactional'='true', 'transactional_properties'='insert_only'); +select * from smt1_tab2 where c1 = '0'; + +-- Temp table still works +create temporary table smt1_tmp (c1 string, c2 string) stored as orc; +insert into table smt1_tmp values ('123', '456'); + +select * from smt1_tmp; + +select c1, count(*) from smt1_tmp group by c1; + http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/queries/clientpositive/strict_managed_tables2.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/strict_managed_tables2.q b/ql/src/test/queries/clientpositive/strict_managed_tables2.q new file mode 100644 index 0000000..72479f0 --- /dev/null +++ b/ql/src/test/queries/clientpositive/strict_managed_tables2.q @@ -0,0 +1,46 @@ + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +set metastore.strict.managed.tables=true; +set hive.default.fileformat=textfile; +set hive.default.fileformat.managed=orc; + +set metastore.create.as.acid=true; +set hive.create.as.insert.only=false; + +drop table if exists smt2_tab1; +drop table if exists smt2_tab2; +drop table if exists smt2_tab3; +drop table if exists smt2_tab4; +drop table if exists smt2_tab5; +drop table if exists smt2_tab6; + +create external table smt2_tab1 (c1 string, c2 string); +show create table smt2_tab1; + +create table smt2_tab2 (c1 string, c2 string); +show create table smt2_tab2; + +create table smt2_tab3 (c1 string, c2 string) stored as orc; +show create table smt2_tab3; + +set metastore.create.as.acid=false; +set hive.create.as.insert.only=true; + +create external table smt2_tab4 (c1 string, c2 string) stored as orc; +show create table smt2_tab4; + +create table smt2_tab5 (c1 string, c2 string); +show create table smt2_tab5; + +create table smt2_tab6 (c1 string, c2 string) stored as textfile; +show create table smt2_tab6; + +drop table if exists smt2_tab1; +drop table if exists smt2_tab2; +drop table if exists smt2_tab3; +drop table if exists smt2_tab4; +drop table if exists smt2_tab5; +drop table if exists smt2_tab6; + http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/queries/clientpositive/strict_managed_tables_sysdb.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/strict_managed_tables_sysdb.q b/ql/src/test/queries/clientpositive/strict_managed_tables_sysdb.q new file mode 100644 index 0000000..76b595b --- /dev/null +++ b/ql/src/test/queries/clientpositive/strict_managed_tables_sysdb.q @@ -0,0 +1,48 @@ +--! qt:dataset:alltypesorc,alltypesparquet,cbo_t1,cbo_t2,cbo_t3,lineitem,part,src,src1,src_cbo,src_json,src_sequencefile,src_thrift,srcbucket,srcbucket2,srcpart +set hive.strict.checks.cartesian.product=false; + +set hive.compute.query.using.stats=false; + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +set hive.cbo.enable=false; +set metastore.strict.managed.tables=true; + + +create external table smt_sysdb_src_buck (key int, value string) clustered by(value) into 2 buckets; + +create external table smt_sysdb_src_skew (key int) skewed by (key) on (1,2,3); + +CREATE TABLE smt_sysdb_scr_txn (key int, value string) + CLUSTERED BY (key) INTO 2 BUCKETS STORED AS ORC + TBLPROPERTIES ( + "transactional"="true", + "compactor.mapreduce.map.memory.mb"="2048", + "compactorthreshold.hive.compactor.delta.num.threshold"="4", + "compactorthreshold.hive.compactor.delta.pct.threshold"="0.5"); + +CREATE TEMPORARY TABLE smt_sysdb_src_tmp (key int, value string); + +CREATE EXTERNAL TABLE smt_sysdb_moretypes (a decimal(10,2), b tinyint, c smallint, d int, e bigint, f varchar(10), g char(3)); + +CREATE VIEW smt_sysdb_view + AS select smt_sysdb_src_buck.key, smt_sysdb_scr_txn.value + from smt_sysdb_src_buck, smt_sysdb_scr_txn where smt_sysdb_src_buck.key = smt_sysdb_scr_txn.key; + +show grant user hive_test_user; + +source ../../metastore/scripts/upgrade/hive/hive-schema-3.1.0.hive.sql; + +use sys; + +select tbl_name, tbl_type from tbls where tbl_name like 'smt_sysdb%' order by tbl_name; + +drop table smt_sysdb_src_buck; +drop table smt_sysdb_src_skew; +drop table smt_sysdb_scr_txn; +drop table smt_sysdb_src_tmp; +drop table smt_sysdb_moretypes; + +DROP DATABASE IF EXISTS SYS CASCADE; +DROP DATABASE IF EXISTS INFORMATION_SCHEMA CASCADE; http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/results/clientnegative/strict_managed_tables1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/strict_managed_tables1.q.out b/ql/src/test/results/clientnegative/strict_managed_tables1.q.out new file mode 100644 index 0000000..a659644 --- /dev/null +++ b/ql/src/test/results/clientnegative/strict_managed_tables1.q.out @@ -0,0 +1,29 @@ +PREHOOK: query: create table strict_managed_tables1_tab1 (c1 string, c2 string) stored as orc tblproperties ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@strict_managed_tables1_tab1 +POSTHOOK: query: create table strict_managed_tables1_tab1 (c1 string, c2 string) stored as orc tblproperties ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@strict_managed_tables1_tab1 +PREHOOK: query: create table strict_managed_tables1_tab2 (c1 string, c2 string) stored as textfile tblproperties ('transactional'='true', 'transactional_properties'='insert_only') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@strict_managed_tables1_tab2 +POSTHOOK: query: create table strict_managed_tables1_tab2 (c1 string, c2 string) stored as textfile tblproperties ('transactional'='true', 'transactional_properties'='insert_only') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@strict_managed_tables1_tab2 +PREHOOK: query: create external table strict_managed_tables1_tab3 (c1 string, c2 string) stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@strict_managed_tables1_tab3 +POSTHOOK: query: create external table strict_managed_tables1_tab3 (c1 string, c2 string) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@strict_managed_tables1_tab3 +PREHOOK: query: create table strict_managed_tables1_tab4 (c1 string, c2 string) stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@strict_managed_tables1_tab4 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Table default.strict_managed_tables1_tab4 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional.) http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/results/clientnegative/strict_managed_tables2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/strict_managed_tables2.q.out b/ql/src/test/results/clientnegative/strict_managed_tables2.q.out new file mode 100644 index 0000000..85753a4 --- /dev/null +++ b/ql/src/test/results/clientnegative/strict_managed_tables2.q.out @@ -0,0 +1,13 @@ +PREHOOK: query: create external table strict_managed_tables2_tab1 (c1 string, c2 string) stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@strict_managed_tables2_tab1 +POSTHOOK: query: create external table strict_managed_tables2_tab1 (c1 string, c2 string) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@strict_managed_tables2_tab1 +PREHOOK: query: alter table strict_managed_tables2_tab1 set tblproperties ('EXTERNAL'='FALSE') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@strict_managed_tables2_tab1 +PREHOOK: Output: default@strict_managed_tables2_tab1 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Table default.strict_managed_tables2_tab1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional. http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/results/clientnegative/strict_managed_tables3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/strict_managed_tables3.q.out b/ql/src/test/results/clientnegative/strict_managed_tables3.q.out new file mode 100644 index 0000000..7de8a80 --- /dev/null +++ b/ql/src/test/results/clientnegative/strict_managed_tables3.q.out @@ -0,0 +1,13 @@ +PREHOOK: query: create external table strict_managed_tables3_tab1 (c1 string, c2 string) stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@strict_managed_tables3_tab1 +POSTHOOK: query: create external table strict_managed_tables3_tab1 (c1 string, c2 string) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@strict_managed_tables3_tab1 +PREHOOK: query: alter table strict_managed_tables3_tab1 unset tblproperties ('EXTERNAL') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@strict_managed_tables3_tab1 +PREHOOK: Output: default@strict_managed_tables3_tab1 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Table default.strict_managed_tables3_tab1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional. http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/results/clientnegative/strict_managed_tables4.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/strict_managed_tables4.q.out b/ql/src/test/results/clientnegative/strict_managed_tables4.q.out new file mode 100644 index 0000000..0c7576f --- /dev/null +++ b/ql/src/test/results/clientnegative/strict_managed_tables4.q.out @@ -0,0 +1,31 @@ +PREHOOK: query: CREATE EXTERNAL TABLE strict_managed_tables6_tab1 +STORED AS AVRO +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@strict_managed_tables6_tab1 +POSTHOOK: query: CREATE EXTERNAL TABLE strict_managed_tables6_tab1 +STORED AS AVRO +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@strict_managed_tables6_tab1 +PREHOOK: query: describe strict_managed_tables6_tab1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@strict_managed_tables6_tab1 +POSTHOOK: query: describe strict_managed_tables6_tab1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@strict_managed_tables6_tab1 +col1 string +col2 string +col3 double +col4 string +col5 string +col6 int +PREHOOK: query: CREATE TABLE strict_managed_tables6_tab2 +STORED AS AVRO +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@strict_managed_tables6_tab2 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Table default.strict_managed_tables6_tab2 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional.) http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/results/clientnegative/strict_managed_tables5.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/strict_managed_tables5.q.out b/ql/src/test/results/clientnegative/strict_managed_tables5.q.out new file mode 100644 index 0000000..0e29fbd --- /dev/null +++ b/ql/src/test/results/clientnegative/strict_managed_tables5.q.out @@ -0,0 +1,19 @@ +PREHOOK: query: create external table strict_managed_tables5_tab1 (c1 string, c2 string) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@strict_managed_tables5_tab1 +POSTHOOK: query: create external table strict_managed_tables5_tab1 (c1 string, c2 string) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@strict_managed_tables5_tab1 +PREHOOK: query: create table strict_managed_tables5_tab2 (c1 string, c2 string) +ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@strict_managed_tables5_tab2 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Table default.strict_managed_tables5_tab2 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional.) http://git-wip-us.apache.org/repos/asf/hive/blob/c96e9edd/ql/src/test/results/clientpositive/llap/resourceplan.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/resourceplan.q.out b/ql/src/test/results/clientpositive/llap/resourceplan.q.out index bd55592..143b710 100644 --- a/ql/src/test/results/clientpositive/llap/resourceplan.q.out +++ b/ql/src/test/results/clientpositive/llap/resourceplan.q.out @@ -82,7 +82,7 @@ PREHOOK: Input: database:sys POSTHOOK: query: USE SYS POSTHOOK: type: SWITCHDATABASE POSTHOOK: Input: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `BUCKETING_COLS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `BUCKETING_COLS` ( `SD_ID` bigint, `BUCKET_COL_NAME` string, `INTEGER_IDX` int, @@ -102,7 +102,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@BUCKETING_COLS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `BUCKETING_COLS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `BUCKETING_COLS` ( `SD_ID` bigint, `BUCKET_COL_NAME` string, `INTEGER_IDX` int, @@ -122,7 +122,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@BUCKETING_COLS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `CDS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `CDS` ( `CD_ID` bigint, CONSTRAINT `SYS_PK_CDS` PRIMARY KEY (`CD_ID`) DISABLE ) @@ -138,7 +138,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@CDS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `CDS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `CDS` ( `CD_ID` bigint, CONSTRAINT `SYS_PK_CDS` PRIMARY KEY (`CD_ID`) DISABLE ) @@ -154,7 +154,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@CDS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `COLUMNS_V2` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `COLUMNS_V2` ( `CD_ID` bigint, `COMMENT` string, `COLUMN_NAME` string, @@ -178,7 +178,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@COLUMNS_V2 PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `COLUMNS_V2` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `COLUMNS_V2` ( `CD_ID` bigint, `COMMENT` string, `COLUMN_NAME` string, @@ -202,7 +202,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@COLUMNS_V2 POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `DATABASE_PARAMS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `DATABASE_PARAMS` ( `DB_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -222,7 +222,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@DATABASE_PARAMS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `DATABASE_PARAMS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `DATABASE_PARAMS` ( `DB_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -242,7 +242,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@DATABASE_PARAMS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `DBS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `DBS` ( `DB_ID` bigint, `DB_LOCATION_URI` string, `NAME` string, @@ -266,7 +266,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@DBS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `DBS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `DBS` ( `DB_ID` bigint, `DB_LOCATION_URI` string, `NAME` string, @@ -290,7 +290,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@DBS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `DB_PRIVS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `DB_PRIVS` ( `DB_GRANT_ID` bigint, `CREATE_TIME` int, `DB_ID` bigint, @@ -324,7 +324,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@DB_PRIVS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `DB_PRIVS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `DB_PRIVS` ( `DB_GRANT_ID` bigint, `CREATE_TIME` int, `DB_ID` bigint, @@ -358,7 +358,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@DB_PRIVS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `GLOBAL_PRIVS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `GLOBAL_PRIVS` ( `USER_GRANT_ID` bigint, `CREATE_TIME` int, `GRANT_OPTION` string, @@ -390,7 +390,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@GLOBAL_PRIVS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `GLOBAL_PRIVS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `GLOBAL_PRIVS` ( `USER_GRANT_ID` bigint, `CREATE_TIME` int, `GRANT_OPTION` string, @@ -422,7 +422,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@GLOBAL_PRIVS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `PARTITIONS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITIONS` ( `PART_ID` bigint, `CREATE_TIME` int, `LAST_ACCESS_TIME` int, @@ -448,7 +448,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@PARTITIONS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `PARTITIONS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITIONS` ( `PART_ID` bigint, `CREATE_TIME` int, `LAST_ACCESS_TIME` int, @@ -474,7 +474,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@PARTITIONS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `PARTITION_KEYS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITION_KEYS` ( `TBL_ID` bigint, `PKEY_COMMENT` string, `PKEY_NAME` string, @@ -498,7 +498,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@PARTITION_KEYS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `PARTITION_KEYS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITION_KEYS` ( `TBL_ID` bigint, `PKEY_COMMENT` string, `PKEY_NAME` string, @@ -522,7 +522,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@PARTITION_KEYS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `PARTITION_KEY_VALS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITION_KEY_VALS` ( `PART_ID` bigint, `PART_KEY_VAL` string, `INTEGER_IDX` int, @@ -542,7 +542,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@PARTITION_KEY_VALS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `PARTITION_KEY_VALS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITION_KEY_VALS` ( `PART_ID` bigint, `PART_KEY_VAL` string, `INTEGER_IDX` int, @@ -562,7 +562,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@PARTITION_KEY_VALS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `PARTITION_PARAMS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITION_PARAMS` ( `PART_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -582,7 +582,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@PARTITION_PARAMS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `PARTITION_PARAMS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PARTITION_PARAMS` ( `PART_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -602,7 +602,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@PARTITION_PARAMS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `PART_COL_PRIVS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PART_COL_PRIVS` ( `PART_COLUMN_GRANT_ID` bigint, `COLUMN_NAME` string, `CREATE_TIME` int, @@ -638,7 +638,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@PART_COL_PRIVS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `PART_COL_PRIVS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PART_COL_PRIVS` ( `PART_COLUMN_GRANT_ID` bigint, `COLUMN_NAME` string, `CREATE_TIME` int, @@ -674,7 +674,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@PART_COL_PRIVS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `PART_PRIVS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PART_PRIVS` ( `PART_GRANT_ID` bigint, `CREATE_TIME` int, `GRANT_OPTION` int, @@ -708,7 +708,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@PART_PRIVS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `PART_PRIVS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PART_PRIVS` ( `PART_GRANT_ID` bigint, `CREATE_TIME` int, `GRANT_OPTION` int, @@ -742,7 +742,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@PART_PRIVS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `ROLES` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `ROLES` ( `ROLE_ID` bigint, `CREATE_TIME` int, `OWNER_NAME` string, @@ -764,7 +764,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@ROLES PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `ROLES` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `ROLES` ( `ROLE_ID` bigint, `CREATE_TIME` int, `OWNER_NAME` string, @@ -786,7 +786,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@ROLES POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `ROLE_MAP` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `ROLE_MAP` ( `ROLE_GRANT_ID` bigint, `ADD_TIME` int, `GRANT_OPTION` int, @@ -816,7 +816,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@ROLE_MAP PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `ROLE_MAP` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `ROLE_MAP` ( `ROLE_GRANT_ID` bigint, `ADD_TIME` int, `GRANT_OPTION` int, @@ -846,7 +846,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@ROLE_MAP POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `SDS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SDS` ( `SD_ID` bigint, `CD_ID` bigint, `INPUT_FORMAT` string, @@ -878,7 +878,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@SDS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `SDS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SDS` ( `SD_ID` bigint, `CD_ID` bigint, `INPUT_FORMAT` string, @@ -910,7 +910,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@SDS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `SD_PARAMS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SD_PARAMS` ( `SD_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -930,7 +930,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@SD_PARAMS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `SD_PARAMS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SD_PARAMS` ( `SD_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -950,7 +950,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@SD_PARAMS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `SEQUENCE_TABLE` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SEQUENCE_TABLE` ( `SEQUENCE_NAME` string, `NEXT_VAL` bigint, CONSTRAINT `SYS_PK_SEQUENCE_TABLE` PRIMARY KEY (`SEQUENCE_NAME`) DISABLE @@ -968,7 +968,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@SEQUENCE_TABLE PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `SEQUENCE_TABLE` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SEQUENCE_TABLE` ( `SEQUENCE_NAME` string, `NEXT_VAL` bigint, CONSTRAINT `SYS_PK_SEQUENCE_TABLE` PRIMARY KEY (`SEQUENCE_NAME`) DISABLE @@ -986,7 +986,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@SEQUENCE_TABLE POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `SERDES` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SERDES` ( `SERDE_ID` bigint, `NAME` string, `SLIB` string, @@ -1006,7 +1006,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@SERDES PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `SERDES` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SERDES` ( `SERDE_ID` bigint, `NAME` string, `SLIB` string, @@ -1026,7 +1026,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@SERDES POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `SERDE_PARAMS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SERDE_PARAMS` ( `SERDE_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -1046,7 +1046,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@SERDE_PARAMS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `SERDE_PARAMS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SERDE_PARAMS` ( `SERDE_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -1066,7 +1066,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@SERDE_PARAMS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `SKEWED_COL_NAMES` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_COL_NAMES` ( `SD_ID` bigint, `SKEWED_COL_NAME` string, `INTEGER_IDX` int, @@ -1086,7 +1086,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@SKEWED_COL_NAMES PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `SKEWED_COL_NAMES` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_COL_NAMES` ( `SD_ID` bigint, `SKEWED_COL_NAME` string, `INTEGER_IDX` int, @@ -1106,7 +1106,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@SKEWED_COL_NAMES POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `SKEWED_COL_VALUE_LOC_MAP` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_COL_VALUE_LOC_MAP` ( `SD_ID` bigint, `STRING_LIST_ID_KID` bigint, `LOCATION` string, @@ -1126,7 +1126,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@SKEWED_COL_VALUE_LOC_MAP PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `SKEWED_COL_VALUE_LOC_MAP` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_COL_VALUE_LOC_MAP` ( `SD_ID` bigint, `STRING_LIST_ID_KID` bigint, `LOCATION` string, @@ -1146,7 +1146,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@SKEWED_COL_VALUE_LOC_MAP POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_STRING_LIST` ( `STRING_LIST_ID` bigint, CONSTRAINT `SYS_PK_SKEWED_STRING_LIST` PRIMARY KEY (`STRING_LIST_ID`) DISABLE ) @@ -1162,7 +1162,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@SKEWED_STRING_LIST PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_STRING_LIST` ( `STRING_LIST_ID` bigint, CONSTRAINT `SYS_PK_SKEWED_STRING_LIST` PRIMARY KEY (`STRING_LIST_ID`) DISABLE ) @@ -1178,7 +1178,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@SKEWED_STRING_LIST POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST_VALUES` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_STRING_LIST_VALUES` ( `STRING_LIST_ID` bigint, `STRING_LIST_VALUE` string, `INTEGER_IDX` int, @@ -1198,7 +1198,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@SKEWED_STRING_LIST_VALUES PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `SKEWED_STRING_LIST_VALUES` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_STRING_LIST_VALUES` ( `STRING_LIST_ID` bigint, `STRING_LIST_VALUE` string, `INTEGER_IDX` int, @@ -1218,7 +1218,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@SKEWED_STRING_LIST_VALUES POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `SKEWED_VALUES` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_VALUES` ( `SD_ID_OID` bigint, `STRING_LIST_ID_EID` bigint, `INTEGER_IDX` int, @@ -1238,7 +1238,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@SKEWED_VALUES PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `SKEWED_VALUES` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SKEWED_VALUES` ( `SD_ID_OID` bigint, `STRING_LIST_ID_EID` bigint, `INTEGER_IDX` int, @@ -1258,7 +1258,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@SKEWED_VALUES POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `SORT_COLS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SORT_COLS` ( `SD_ID` bigint, `COLUMN_NAME` string, `ORDER` int, @@ -1280,7 +1280,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@SORT_COLS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `SORT_COLS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `SORT_COLS` ( `SD_ID` bigint, `COLUMN_NAME` string, `ORDER` int, @@ -1302,7 +1302,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@SORT_COLS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `TABLE_PARAMS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `TABLE_PARAMS` ( `TBL_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -1322,7 +1322,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@TABLE_PARAMS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `TABLE_PARAMS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `TABLE_PARAMS` ( `TBL_ID` bigint, `PARAM_KEY` string, `PARAM_VALUE` string, @@ -1342,7 +1342,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@TABLE_PARAMS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `TBLS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `TBLS` ( `TBL_ID` bigint, `CREATE_TIME` int, `DB_ID` bigint, @@ -1379,7 +1379,7 @@ FROM \"TBLS\"" PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@TBLS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `TBLS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `TBLS` ( `TBL_ID` bigint, `CREATE_TIME` int, `DB_ID` bigint, @@ -1416,7 +1416,7 @@ FROM \"TBLS\"" POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@TBLS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `MV_CREATION_METADATA` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `MV_CREATION_METADATA` ( `MV_CREATION_METADATA_ID` bigint, `DB_NAME` string, `TBL_NAME` string, @@ -1437,7 +1437,7 @@ FROM \"MV_CREATION_METADATA\"" PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@MV_CREATION_METADATA PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `MV_CREATION_METADATA` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `MV_CREATION_METADATA` ( `MV_CREATION_METADATA_ID` bigint, `DB_NAME` string, `TBL_NAME` string, @@ -1458,7 +1458,7 @@ FROM \"MV_CREATION_METADATA\"" POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@MV_CREATION_METADATA POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `MV_TABLES_USED` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `MV_TABLES_USED` ( `MV_CREATION_METADATA_ID` bigint, `TBL_ID` bigint ) @@ -1474,7 +1474,7 @@ FROM \"MV_TABLES_USED\"" PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@MV_TABLES_USED PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `MV_TABLES_USED` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `MV_TABLES_USED` ( `MV_CREATION_METADATA_ID` bigint, `TBL_ID` bigint ) @@ -1490,7 +1490,7 @@ FROM \"MV_TABLES_USED\"" POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@MV_TABLES_USED POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `TBL_COL_PRIVS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `TBL_COL_PRIVS` ( `TBL_COLUMN_GRANT_ID` bigint, `COLUMN_NAME` string, `CREATE_TIME` int, @@ -1526,7 +1526,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@TBL_COL_PRIVS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `TBL_COL_PRIVS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `TBL_COL_PRIVS` ( `TBL_COLUMN_GRANT_ID` bigint, `COLUMN_NAME` string, `CREATE_TIME` int, @@ -1562,7 +1562,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@TBL_COL_PRIVS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `TBL_PRIVS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `TBL_PRIVS` ( `TBL_GRANT_ID` bigint, `CREATE_TIME` int, `GRANT_OPTION` int, @@ -1596,7 +1596,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@TBL_PRIVS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `TBL_PRIVS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `TBL_PRIVS` ( `TBL_GRANT_ID` bigint, `CREATE_TIME` int, `GRANT_OPTION` int, @@ -1630,7 +1630,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@TBL_PRIVS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `TAB_COL_STATS` ( `CS_ID` bigint, `DB_NAME` string, `TABLE_NAME` string, @@ -1682,7 +1682,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@TAB_COL_STATS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `TAB_COL_STATS` ( `CS_ID` bigint, `DB_NAME` string, `TABLE_NAME` string, @@ -1734,7 +1734,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@TAB_COL_STATS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `PART_COL_STATS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PART_COL_STATS` ( `CS_ID` bigint, `DB_NAME` string, `TABLE_NAME` string, @@ -1788,7 +1788,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@PART_COL_STATS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `PART_COL_STATS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `PART_COL_STATS` ( `CS_ID` bigint, `DB_NAME` string, `TABLE_NAME` string, @@ -1842,7 +1842,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@PART_COL_STATS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `VERSION` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `VERSION` ( `VER_ID` BIGINT, `SCHEMA_VERSION` string, `VERSION_COMMENT` string, @@ -1851,7 +1851,7 @@ PREHOOK: query: CREATE TABLE IF NOT EXISTS `VERSION` ( PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@VERSION PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `VERSION` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `VERSION` ( `VER_ID` BIGINT, `SCHEMA_VERSION` string, `VERSION_COMMENT` string, @@ -1871,7 +1871,7 @@ POSTHOOK: Output: sys@version POSTHOOK: Lineage: version.schema_version SCRIPT [] POSTHOOK: Lineage: version.ver_id SCRIPT [] POSTHOOK: Lineage: version.version_comment SCRIPT [] -PREHOOK: query: CREATE TABLE IF NOT EXISTS `DB_VERSION` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `DB_VERSION` ( `VER_ID` BIGINT, `SCHEMA_VERSION` string, `VERSION_COMMENT` string, @@ -1891,7 +1891,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@DB_VERSION PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `DB_VERSION` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `DB_VERSION` ( `VER_ID` BIGINT, `SCHEMA_VERSION` string, `VERSION_COMMENT` string, @@ -1911,7 +1911,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@DB_VERSION POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `FUNCS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `FUNCS` ( `FUNC_ID` bigint, `CLASS_NAME` string, `CREATE_TIME` int, @@ -1941,7 +1941,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@FUNCS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `FUNCS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `FUNCS` ( `FUNC_ID` bigint, `CLASS_NAME` string, `CREATE_TIME` int, @@ -1971,7 +1971,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@FUNCS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `KEY_CONSTRAINTS` +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `KEY_CONSTRAINTS` ( `CHILD_CD_ID` bigint, `CHILD_INTEGER_IDX` int, @@ -2012,7 +2012,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@KEY_CONSTRAINTS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `KEY_CONSTRAINTS` +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `KEY_CONSTRAINTS` ( `CHILD_CD_ID` bigint, `CHILD_INTEGER_IDX` int, @@ -2123,7 +2123,7 @@ POSTHOOK: Lineage: PARTITION_STATS_VIEW.part_id SIMPLE [(partition_params)partit POSTHOOK: Lineage: PARTITION_STATS_VIEW.raw_data_size EXPRESSION [(partition_params)partition_params.FieldSchema(name:param_key, type:string, comment:from deserializer), (partition_params)partition_params.FieldSchema(name:param_value, type:string, comment:from deserializer), ] POSTHOOK: Lineage: PARTITION_STATS_VIEW.total_size EXPRESSION [(partition_params)partition_params.FieldSchema(name:param_key, type:string, comment:from deserializer), (partition_params)partition_params.FieldSchema(name:param_value, type:string, comment:from deserializer), ] POSTHOOK: Lineage: PARTITION_STATS_VIEW.transient_last_ddl_time EXPRESSION [(partition_params)partition_params.FieldSchema(name:param_key, type:string, comment:from deserializer), (partition_params)partition_params.FieldSchema(name:param_value, type:string, comment:from deserializer), ] -PREHOOK: query: CREATE TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( `NAME` string, `STATUS` string, `QUERY_PARALLELISM` int, @@ -2144,7 +2144,7 @@ FROM PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@WM_RESOURCEPLANS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_RESOURCEPLANS` ( `NAME` string, `STATUS` string, `QUERY_PARALLELISM` int, @@ -2165,7 +2165,7 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@WM_RESOURCEPLANS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `WM_TRIGGERS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_TRIGGERS` ( `RP_NAME` string, `NAME` string, `TRIGGER_EXPRESSION` string, @@ -2190,7 +2190,7 @@ ON PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@WM_TRIGGERS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `WM_TRIGGERS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_TRIGGERS` ( `RP_NAME` string, `NAME` string, `TRIGGER_EXPRESSION` string, @@ -2215,7 +2215,7 @@ ON POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@WM_TRIGGERS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `WM_POOLS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS` ( `RP_NAME` string, `PATH` string, `ALLOC_FRACTION` double, @@ -2242,7 +2242,7 @@ ON PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@WM_POOLS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `WM_POOLS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS` ( `RP_NAME` string, `PATH` string, `ALLOC_FRACTION` double, @@ -2269,7 +2269,7 @@ ON POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@WM_POOLS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `WM_POOLS_TO_TRIGGERS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS_TO_TRIGGERS` ( `RP_NAME` string, `POOL_PATH` string, `TRIGGER_NAME` string @@ -2299,7 +2299,7 @@ WHERE CAST(\"WM_TRIGGER\".\"IS_IN_UNMANAGED\" AS CHAR) IN ('1', 't') PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@WM_POOLS_TO_TRIGGERS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `WM_POOLS_TO_TRIGGERS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_POOLS_TO_TRIGGERS` ( `RP_NAME` string, `POOL_PATH` string, `TRIGGER_NAME` string @@ -2329,7 +2329,7 @@ WHERE CAST(\"WM_TRIGGER\".\"IS_IN_UNMANAGED\" AS CHAR) IN ('1', 't') POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@WM_POOLS_TO_TRIGGERS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `WM_MAPPINGS` ( +PREHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_MAPPINGS` ( `RP_NAME` string, `ENTITY_TYPE` string, `ENTITY_NAME` string, @@ -2354,7 +2354,7 @@ LEFT OUTER JOIN \"WM_POOL\" ON \"WM_POOL\".\"POOL_ID\" = \"WM_MAPPING\".\"POOL_I PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@WM_MAPPINGS PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `WM_MAPPINGS` ( +POSTHOOK: query: CREATE EXTERNAL TABLE IF NOT EXISTS `WM_MAPPINGS` ( `RP_NAME` string, `ENTITY_TYPE` string, `ENTITY_NAME` string,
