http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/regress/hive/TEST008 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST008 b/core/sql/regress/hive/TEST008 new file mode 100644 index 0000000..5333de3 --- /dev/null +++ b/core/sql/regress/hive/TEST008 @@ -0,0 +1,304 @@ +-- Tests for Hive DDL from Traf +-- +-- @@@ START COPYRIGHT @@@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- +-- @@@ END COPYRIGHT @@@ + +drop schema hive.sch008 cascade; +revoke role DB__HIVEROLE from SQL_USER4; + +log LOG008 clear; +cqd traf_ddl_on_hive_objects 'ON'; + +showddl schema hive.sch008; +create database hive.sch008; +create schema if not exists hive.sch008; +showddl schema hive.sch008; + +drop table if exists hive.sch008.t00801; +create table hive.sch008.t00801 (a int, b int); +invoke hive.sch008.t00801; +showddl hive.sch008.t00801; +drop table hive.sch008.t00801; + +set schema hive.sch008; +create table t00802 (a int, b int); +invoke t00802; + +insert into t00802 values (1,2); +select * from t00802; + +create table t00803 as select * from sch008.t00802; +invoke t00803; +select * from t00803; + +-- describe will show datatype 132 for b +prepare s from insert into t00802 values (?, ?); +describe s; + +-- alter will do QI and reload table definition +alter table t00802 change b b bigint; + +select * from t00802; + +-- describe will show datatype 134 for b +prepare s from insert into t00802 values (?, ?); +describe s; + +invoke t00802; +showddl t00802; + +-- create in hive default schema +drop table if exists hive.`default`.t00804; +create table hive.`default`.`t00804` (`a` int); +showddl hive.hive.t00804; +insert into hive.hive.t00804 values (11); + +create view if not exists hive.sch008.v00802 as select * from sch008.t00802; +showddl hive.sch008.v00802; +select * from v00802; +alter view hive.sch008.v00802 as select * from sch008.t00802; +showddl hive.sch008.v00802; + +-- view in a different schema than table +create view hive.sch008.v00803 as select * from `default`.t00804; +showddl hive.sch008.v00803; +prepare s from select * from v00803; + +get objects in schema hive.sch008; + +drop view if exists hive.sch008.v00802; +invoke hive.sch008.v00802; + +get objects in schema hive.sch008; + +-- delimited names using hive ` as delimiter. +create table hive.`sch008`.`t00804` (`a` int); +showddl hive.sch008.t00804; +create table hive.`sch008`.`t00804like` like `sch008`.`t00804`; +create external table hive.`sch008`.`t00804like2` like `sch008`.`t00804`; +showddl hive.sch008.t00804like; +showddl hive.sch008.t00804like2; +drop table if exists hive.sch008.`t00805like`; +drop table if exists hive.sch008.`t00805like2`; + +-- delimited names using traf " as delimiter. +create table hive."sch008"."t00805" (`a` int); +showddl hive.sch008.t00805; + +-- create external hive table LIKE a traf table +drop table if exists hive.sch008.columns; +create external hive table hive.sch008.columns like trafodion."_MD_".columns + with hive options 'stored as sequencefile'; +create external hive table if not exists hive.sch008.columns like trafodion."_MD_".columns + with hive options 'stored as sequencefile'; +showddl hive.sch008.columns; +-- create managed hive table LIKE traf table +drop table if exists hive.sch008.columns; +create hive table hive.sch008.columns like trafodion."_MD_".columns + with hive options 'stored as sequencefile'; +showddl hive.sch008.columns; + +-- create and drop hive table and external table +drop table if exists t00806; +create table t00806 (a int, b string); +showddl t00806; +create external table t00806 for t00806; +showddl t00806; +drop table if exists t00806; +showddl t00806; +select object_uid from trafodion."_MD_".objects where object_name = 'T00806'; + +-- cleanup hive table +drop table if exists t00806; +create table t00806 (a int, b string); +showddl t00806; +create external table t00806 for t00806; +showddl t00806; +cleanup table t00806; +showddl t00806; +select trim(schema_name) || '.' || trim(object_name) from trafodion."_MD_".objects where object_name = 'T00806'; +drop table if exists t00806; +create table t00806 (a int, b string); +create external table t00806 for t00806; +process hive ddl 'drop table sch008.t00806'; +showddl t00806; +select trim(schema_name) || '.' || trim(object_name) from trafodion."_MD_".objects where object_name = 'T00806'; +cleanup table t00806; +showddl t00806; +select trim(schema_name) || '.' || trim(object_name) from trafodion."_MD_".objects where object_name = 'T00806'; + +-- TRUNCATE TABLE tests +-- hive managed table +create table if not exists t00807 (a int, b int); +showddl t00807; +insert into t00807 values (1,2), (3,4); +select * from t00807; +truncate table t00807; +select * from t00807; +drop table t00807; +-- next truncate should return error +truncate table t00807; +truncate table if exists t00807; + +-- hive external table +drop table t00807; +create external table if not exists t00807 (a int, b int); +showddl t00807; +insert into t00807 values (1,2), (3,4); +select * from t00807; +truncate table t00807; +select * from t00807; +drop table t00807; + +-- hive partitioned external table. +-- hive partitioned table insert/select not yet supported from traf interface. +-- When supported, remove regrhive.ksh and do the insert/select from traf. +-- Also enabled select from t00807. +drop table t00807; +create external table if not exists t00807 (a int) partitioned by (b int); + +-- TBD: insert into sch008.t00807 values (1,2); +sh echo "insert into sch008.t00807 partition(b=2) values (1);" > TEST008_junk; +sh regrhive.ksh -f TEST008_junk; + +-- TBD: select * from t00807; +sh echo "select * from sch008.t00807;" > TEST008_junk; +sh regrhive.ksh -f TEST008_junk | tee -a LOG008; + +truncate table t00807; + +-- TBD: select * from t00807; +sh echo "select * from sch008.t00807;" > TEST008_junk; +sh regrhive.ksh -f TEST008_junk | tee -a LOG008; + +drop table t00807; + + +-- explain on hive DDL +explain options 'f' drop table hive.hive.tnone; +explain options 'f' create table hive.hive.tnone (a bigint); + +-- cannot access hive objects if not DB__HIVEROLE. +-- should return error, SQL_USER4 not authorized. +log; +sh sqlci -i"TEST008(hive_ddl_as_user4)" -u"SQL_User4"; +log LOG008; + +-- grant/revoke to access hive objects +showddl role DB__HIVEROLE; +grant role DB__HIVEROLE to SQL_USER4; +showddl role DB__HIVEROLE; + +-- should not return unquthorized error. +log; +sh sqlci -i"TEST008(hive_ddl_as_user4)" -u"SQL_User4"; +log LOG008; + +revoke role DB__HIVEROLE from SQL_USER4; +showddl role DB__HIVEROLE; + +-- should return error +log; +sh sqlci -i"TEST008(hive_ddl_as_user4)" -u"SQL_User4"; + +log LOG008; + +-- next alter should be passed on to hive. will return error from hive. +alter schema hive.sch008 abc; +alter database hive.sch008 abc; + +-- error cases +set catalog hive; +drop table if exists sch008.temptab; + +-- next drop should return error but hive doesn't return error +-- on a nonexistent table +process hive ddl 'drop table sch008.temptab'; + +drop table if exists sch008.temptab; + +create table sch008.temptab (a int); + +-- next create should return error that table exists +create table sch008.temptab (a int); +create table if not exists sch008.temptab (a int); + +set schema hive.sch008; + +-- create table error. largeint is not a valid type. +create table temptab2(a largeint); + +-- return error: schema not empty +drop schema sch008; + +-- use of backquote should return an error for non-hive ddl +create table trafodion.sch.`temp` (a int); +create external table trafodion.sch.`temp` (a int); +create table hive.hive.`temp` for hive.hive.`temp`; +create external table hive.hive.`temp` for hive.hive.`temp`; +create table hive.hive.`temp` like hive.hive.`temp`; +select * from `dual`; + +-- cannot use keyword database to create/drop traf schemas +create database trafodion.temp; +drop database trafodion.temp; +alter database trafodion.temp; + +-- return error: unsupported operation from 'process hive ddl' stmt +process hive ddl 'grnt abc'; + +-- return error: ctas on hive cannot use traf in the SELECT query +create table hive.hive.temptab2 as select * from trafodion.sch.temp; + +-- cannot use 'with hive options' with non-hive tables +create table trafodion.seabase.temp like trafodion."_MD_".objects + with hive options 'abc'; + +-- target table must be a hive table +create external hive table trafodion.seabase.temp like trafodion."_MD_".objects; + +-- source table must be a trafodion table +create external hive table hive.hive.temp like hive.hive.temp2 + with hive options 'abc'; + +-- ctas tgt cannot be an external hive table +create external hive table hive.hive.temp as select * from trafodion."_MD_".objects; + +-- like option must be specified +create external hive table hive.hive.temp (a int); + +drop database hive.sch008 cascade; +drop schema if exists hive.sch008 cascade; +showddl schema hive.sch008; + +drop table hive.`default`.t00804; +drop table hive.sch008.temptab; + +revoke role DB__HIVEROLE from SQL_USER4; + +log; +exit; + +?section hive_ddl_as_user4 +log LOG008; +process hive ddl 'drop table tnotexists'; +drop table hive.hive.tnotexists; +
http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/regress/hive/TEST009 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST009 b/core/sql/regress/hive/TEST009 index 991cba8..4949d12 100755 --- a/core/sql/regress/hive/TEST009 +++ b/core/sql/regress/hive/TEST009 @@ -94,7 +94,7 @@ process hive statement 'drop view t009cust'; -- The version of hive installed does not support special characters -- TDB - when hive is upgraded to a new version, add tests where -- the hive schema contains special characters -sh regrhive.ksh -v -f $REGRTSTDIR/TEST009_a.hive.sql &> $REGRRUNDIR/LOG009_a.hive.log; +sh sqlci -i"$REGRTSTDIR/TEST009_a.hive.sql" &> $REGRRUNDIR/LOG009_a.hive.log; sh regrhbase.ksh $REGRTSTDIR/TEST009_create_hbase_objects.hbase &> $REGRRUNDIR/LOG009_create_hbase_tables.log ; -- make sure external schemas exist in Trafodion @@ -185,7 +185,7 @@ drop external table item for hive.hive.item; -- cleanup data from the old table, and create/load data with additional column sh regrhadoop.ksh fs -rm /user/trafodion/hive/exttables/t009t1/*; -sh regrhive.ksh -v -f $REGRTSTDIR/TEST009_b.hive.sql &> $REGRRUNDIR/LOG009_b.hive.log;; +sh sqlci -i"$REGRTSTDIR/TEST009_b.hive.sql" &> $REGRRUNDIR/LOG009_b.hive.log; -- should fail - column mismatch select count(*) from hive.sch_t009.t009t1; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/regress/hive/TEST009_a.hive.sql ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST009_a.hive.sql b/core/sql/regress/hive/TEST009_a.hive.sql index 390bbbd..a815e9c 100644 --- a/core/sql/regress/hive/TEST009_a.hive.sql +++ b/core/sql/regress/hive/TEST009_a.hive.sql @@ -23,8 +23,8 @@ -- Our version of HIVE does not support special characters. This test should -- be changed to use delimited names once we upgrade HIVE. -create schema if not exists sch_t009; -use sch_t009; +create schema if not exists hive.sch_t009; +set schema hive.sch_t009; drop table t009t1; create external table t009t1 ( @@ -39,7 +39,7 @@ location '/user/trafodion/hive/exttables/t009t1'; -- load command from an existing table. insert into table t009t1 select c_customer_sk, c_birth_day, c_birth_month -from default.customer +from hive.customer limit 10; select * from t009t1; @@ -56,7 +56,7 @@ location '/user/trafodion/hive/exttables/t009t2'; insert into table t009t2 select c_customer_sk, c_birth_day, c_birth_month -from default.customer +from hive.customer limit 10; select * from t009t2; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/regress/hive/TEST009_b.hive.sql ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST009_b.hive.sql b/core/sql/regress/hive/TEST009_b.hive.sql index b9d088d..e0a0ff5 100644 --- a/core/sql/regress/hive/TEST009_b.hive.sql +++ b/core/sql/regress/hive/TEST009_b.hive.sql @@ -20,29 +20,15 @@ -- specified by the configuration property hive.metastore.warehouse.dir. -- ============================================================================ --- Our version of HIVE does not support special characters. This test should --- be changed to use delimited names once we upgrade HIVE. - -create schema if not exists sch_t009; -use sch_t009; -drop table t009t1; -create external table t009t1 -( - a int, - b int, - c int, - d int -) -row format delimited fields terminated by '|' -location '/user/trafodion/hive/exttables/t009t1'; +process hive ddl 'create schema if not exists sch_t009'; +process hive ddl 'drop table sch_t009.t009t1'; +process hive ddl 'create external table sch_t009.t009t1 (a int, b int, c int, d int) row format delimited fields terminated by ''|'' location ''/user/trafodion/hive/exttables/t009t1'' '; -- Our version of HIVE does not support insert ... VALUES clause, so use the -- load command from an existing table. -insert into table t009t1 -select c_customer_sk, c_birth_day, c_birth_month, c_birth_year -from default.customer -limit 10; +process hive statement 'insert into table sch_t009.t009t1 select c_customer_sk, c_birth_day, c_birth_month, c_birth_year from customer limit 10 '; + +--select * from hive.sch_t009.t009t1; -select * from t009t1; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/regress/hive/TEST018 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST018 b/core/sql/regress/hive/TEST018 index 6b273ec..e793677 100644 --- a/core/sql/regress/hive/TEST018 +++ b/core/sql/regress/hive/TEST018 @@ -34,7 +34,10 @@ obey TEST018(clean_up); log LOG018 clear; -sh regrhive.ksh -v -f $REGRTSTDIR/TEST018_create_hive_tables.hive &> $REGRRUNDIR/LOG018_create_hive_tables.log ; +--sh regrhive.ksh -v -f $REGRTSTDIR/TEST018_create_hive_tables.hive &> $REGRRUNDIR/LOG018_create_hive_tables.log ; +set schema hive.hive; +obey TEST018(create_hive_tables); +set schema trafodion.hbase; obey TEST018(setup); obey TEST018(test_bulk_unload_simple); @@ -249,13 +252,13 @@ select count(*) from store_sales_salt; -- using insert insert overwrite table hive.hive.null_format_default select * from null_format_src; -select * from hive.hive.null_format_default; +select * from hive.hive.null_format_default order by 1,2; insert overwrite table hive.hive.null_format_empty select * from null_format_src; -select * from hive.hive.null_format_empty; +select * from hive.hive.null_format_empty order by 1,2; insert overwrite table hive.hive.null_format_colon select * from null_format_src; -select * from hive.hive.null_format_colon; +select * from hive.hive.null_format_colon order by 1,2; -- using unload @@ -752,3 +755,146 @@ sh regrhadoop.ksh fs -rm /user/trafodion/hive/exttables/unload_customer_demograp log LOG018; log; + +?section create_hive_tables +drop table unload_customer_address; +create external table unload_customer_address +( + ca_address_sk int, + ca_address_id string, + ca_street_number string, + ca_street_name string, + ca_street_type string, + ca_suite_number string, + ca_city string, + ca_county string, + ca_state string, + ca_zip string, + ca_country string, + ca_gmt_offset float, + ca_location_type string +) +row format delimited fields terminated by '|' +location '/user/trafodion/hive/exttables/unload_customer_address'; + + +drop table unload_customer_demographics; +create external table unload_customer_demographics +( + cd_demo_sk int, + cd_gender string, + cd_marital_status string, + cd_education_status string, + cd_purchase_estimate int, + cd_credit_rating string, + cd_dep_count int, + cd_dep_employed_count int, + cd_dep_college_count int +) +row format delimited fields terminated by '|' +location '/user/trafodion/hive/exttables/unload_customer_demographics'; + +drop table unload_customer; +create external table unload_customer +( + c_customer_sk int, + c_customer_id string, + c_current_cdemo_sk int, + c_current_hdemo_sk int, + c_current_addr_sk int, + c_first_shipto_date_sk int, + c_first_sales_date_sk int, + c_salutation string, + c_first_name string, + c_last_name string, + c_preferred_cust_flag string, + c_birth_day int, + c_birth_month int, + c_birth_year int, + c_birth_country string, + c_login string, + c_email_address string, + c_last_review_date string +) +row format delimited fields terminated by '|' LINES TERMINATED BY '\n' +location '/user/trafodion/hive/exttables/unload_customer'; + +drop table unload_customer_name; +create external table unload_customer_name +( + c_first_name string, + c_last_name string +) +row format delimited fields terminated by '|' LINES TERMINATED BY '\n' +location '/user/trafodion/hive/exttables/unload_customer_name'; + +drop table unload_customer_and_address; +create external table unload_customer_and_address +( + c_customer_sk int, + c_customer_id string, + c_current_cdemo_sk int, + c_current_hdemo_sk int, + c_current_addr_sk int, + c_first_shipto_date_sk int, + c_first_sales_date_sk int, + c_salutation string, + c_first_name string, + c_last_name string, + c_preferred_cust_flag string, + c_birth_day int, + c_birth_month int, + c_birth_year int, + c_birth_country string, + c_login string, + c_email_address string, + c_last_review_date string, + ca_address_sk int, + ca_address_id string, + ca_street_number string, + ca_street_name string, + ca_street_type string, + ca_suite_number string, + ca_city string, + ca_county string, + ca_state string, + ca_zip string, + ca_country string, + ca_gmt_offset float, + ca_location_type string +) +row format delimited fields terminated by '|' LINES TERMINATED BY '\n' +location '/user/trafodion/hive/exttables/unload_customer_and_address'; + +drop table tmp_unload_table; +create external table tmp_unload_table +( + filed string +) +row format delimited fields terminated by '|' +location '/user/trafodion/hive/exttables/tmp_unload_table'; + +drop table unload_store_sales_summary; +create external table unload_store_sales_summary +( + ss_sold_date_sk int, + --s_store_id string, + --s_store_name string, + ss_store_sk int, + ss_quantity int +) +row format delimited fields terminated by '|' +location '/user/trafodion/hive/exttables/unload_store_sales_summary'; + +drop table null_format_default; +create external table null_format_default (a string, b string); + +drop table null_format_empty; +create external table null_format_empty(a string, b string) +row format delimited fields terminated by '|' +null defined as ''; + +drop table null_format_colon; +create external table null_format_colon(a string, b string) +row format delimited fields terminated by '|' +null defined as ':'; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/regress/hive/TEST018_create_hive_tables.hive ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST018_create_hive_tables.hive b/core/sql/regress/hive/TEST018_create_hive_tables.hive index 5d60784..630dca1 100644 --- a/core/sql/regress/hive/TEST018_create_hive_tables.hive +++ b/core/sql/regress/hive/TEST018_create_hive_tables.hive @@ -19,7 +19,7 @@ -- -- @@@ END COPYRIGHT @@@ -drop table unload_customer_address; +drop table if exists unload_customer_address; create external table unload_customer_address ( ca_address_sk int, @@ -40,7 +40,7 @@ row format delimited fields terminated by '|' location '/user/trafodion/hive/exttables/unload_customer_address'; -drop table unload_customer_demographics; +drop table if exists unload_customer_demographics; create external table unload_customer_demographics ( cd_demo_sk int, @@ -56,7 +56,7 @@ create external table unload_customer_demographics row format delimited fields terminated by '|' location '/user/trafodion/hive/exttables/unload_customer_demographics'; -drop table unload_customer; +drop table if exists unload_customer; create external table unload_customer ( c_customer_sk int, @@ -81,7 +81,7 @@ create external table unload_customer row format delimited fields terminated by '|' LINES TERMINATED BY '\n' location '/user/trafodion/hive/exttables/unload_customer'; -drop table unload_customer_name; +drop table if exists unload_customer_name; create external table unload_customer_name ( c_first_name string, @@ -90,7 +90,7 @@ create external table unload_customer_name row format delimited fields terminated by '|' LINES TERMINATED BY '\n' location '/user/trafodion/hive/exttables/unload_customer_name'; -drop table unload_customer_and_address; +drop table if exists unload_customer_and_address; create external table unload_customer_and_address ( c_customer_sk int, @@ -128,7 +128,7 @@ create external table unload_customer_and_address row format delimited fields terminated by '|' LINES TERMINATED BY '\n' location '/user/trafodion/hive/exttables/unload_customer_and_address'; -drop table tmp_unload_table; +drop table if exists tmp_unload_table; create external table tmp_unload_table ( filed string @@ -136,7 +136,7 @@ create external table tmp_unload_table row format delimited fields terminated by '|' location '/user/trafodion/hive/exttables/tmp_unload_table'; -drop table unload_store_sales_summary; +drop table if exists unload_store_sales_summary; create external table unload_store_sales_summary ( ss_sold_date_sk int, @@ -148,15 +148,15 @@ create external table unload_store_sales_summary row format delimited fields terminated by '|' location '/user/trafodion/hive/exttables/unload_store_sales_summary'; -drop table null_format_default; +drop table if exists null_format_default; create external table null_format_default (a string, b string); -drop table null_format_empty; +drop table if exists null_format_empty; create external table null_format_empty(a string, b string) row format delimited fields terminated by '|' null defined as ''; -drop table null_format_colon; +drop table if exists null_format_colon; create external table null_format_colon(a string, b string) row format delimited fields terminated by '|' null defined as ':'; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/regress/privs2/EXPECTED129 ---------------------------------------------------------------------- diff --git a/core/sql/regress/privs2/EXPECTED129 b/core/sql/regress/privs2/EXPECTED129 index 5cc90c4..c844a4a 100644 --- a/core/sql/regress/privs2/EXPECTED129 +++ b/core/sql/regress/privs2/EXPECTED129 @@ -608,7 +608,7 @@ X >> >>create view v3bd as select b,d from t3; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. @@ -620,7 +620,7 @@ X >>create view v3b as select b from t3; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. @@ -632,7 +632,7 @@ X >>create view v3d as select d from t3; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. @@ -644,7 +644,7 @@ X >>create view v3bbbbbb (c1,c2,c3,c4,c5,c6) as select b,b,b,b,b,b from t3; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. @@ -659,45 +659,45 @@ X >> >>create view v3ac as select a,c from t3; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. >>create view v3a as select a from t3; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. >>create view v3c as select c from t3; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. >> >>create view v3 as select * from t3; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. >> >>create view v3ab as select a,b from t3; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. >>create view v3abcd as select a,b,c,d from t3; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. >>create view v3bc as select b,c from t3; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. @@ -706,7 +706,7 @@ X >> >>create view v34bf as select b,f from t3, t4; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. @@ -718,7 +718,7 @@ X >>create view v34bdfg as select b,d,f,g from t3, t4; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. @@ -730,7 +730,7 @@ X >>create view v34bdfg2 (c1,c2,c3,c4) as select b,d,f,g from t3, t4; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. @@ -742,7 +742,7 @@ X >>create view v34gb as select g,b from t3, t4; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. @@ -757,19 +757,19 @@ X >> >>create view v34 as select * from t3,t4; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. >>create view v34af as select a,f from t3, t4; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. >>create view v34bh as select b,h from t3,t4; -*** ERROR[3242] This statement is not supported. Reason: DDL views can only be created or dropped in trafodion schema. +*** ERROR[3242] This statement is not supported. Reason: This view cannot be created or dropped in the specified catalog 'CAT'. *** ERROR[8822] The statement was not prepared. http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/regress/seabase/EXPECTED003 ---------------------------------------------------------------------- diff --git a/core/sql/regress/seabase/EXPECTED003 b/core/sql/regress/seabase/EXPECTED003 index 8ce0a4c..95d58b7 100644 --- a/core/sql/regress/seabase/EXPECTED003 +++ b/core/sql/regress/seabase/EXPECTED003 @@ -352,7 +352,7 @@ A B C D >> >>invoke hive.hive.ttiny; --- Definition of hive table TTINY +-- Definition of hive table HIVE.HIVE.TTINY -- Definition current Wed Feb 22 23:04:49 2017 ( @@ -833,7 +833,7 @@ A B C D >> >>invoke hive.hive.ttiny; --- Definition of hive table TTINY +-- Definition of hive table HIVE.HIVE.TTINY -- Definition current Wed Feb 22 23:05:57 2017 ( @@ -2462,7 +2462,7 @@ TRUE FALSE >> >>invoke hive.hive.tbool; --- Definition of hive table TBOOL +-- Definition of hive table HIVE.HIVE.TBOOL -- Definition current Wed Feb 22 23:08:45 2017 ( http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/regress/seabase/EXPECTED031 ---------------------------------------------------------------------- diff --git a/core/sql/regress/seabase/EXPECTED031 b/core/sql/regress/seabase/EXPECTED031 index 6d8d542..0a67a92 100644 --- a/core/sql/regress/seabase/EXPECTED031 +++ b/core/sql/regress/seabase/EXPECTED031 @@ -446,7 +446,7 @@ Z Z (EXPR) (EXPR) >>invoke t031t10; -- Definition of Trafodion table TRAFODION.SCH.T031T10 --- Definition current Sun Mar 11 00:49:14 2018 +-- Definition current Sat Jun 2 02:17:30 2018 ( SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE @@ -630,13 +630,19 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD --- SQL operation complete. >> ->>-- should return error. ->>process hive statement 'insert into t values (1)'; +>>-- should not return error. +>>process hive statement 'drop table t'; -*** ERROR[3242] This statement is not supported. Reason: Only CREATE, DROP, ALTER or TRUNCATE hive DDL statements can be specified. +--- SQL operation complete. +>>process hive statement 'create table t (a int)'; -*** ERROR[8822] The statement was not prepared. +--- SQL operation complete. +>>process hive statement 'insert into t values (1)'; + +--- SQL operation complete. +>>process hive statement 'drop table t'; +--- SQL operation complete. >> >>-- default USER >>drop table if exists t031t1; @@ -648,7 +654,7 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD >>invoke t031t1; -- Definition of Trafodion table TRAFODION.SCH.T031T1 --- Definition current Sun Mar 11 00:50:44 2018 +-- Definition current Sat Jun 2 02:18:43 2018 ( SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE @@ -679,7 +685,7 @@ A B >>invoke t031t1; -- Definition of Trafodion table TRAFODION.SCH.T031T1 --- Definition current Sun Mar 11 00:50:53 2018 +-- Definition current Sat Jun 2 02:18:50 2018 ( SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE @@ -730,7 +736,7 @@ A B C >>invoke t031t1; -- Definition of Trafodion table TRAFODION.SCH.T031T1 --- Definition current Sun Mar 11 00:51:05 2018 +-- Definition current Sat Jun 2 02:18:59 2018 ( SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE @@ -759,7 +765,7 @@ A B C >>invoke t031t1; -- Definition of Trafodion table TRAFODION.SCH.T031T1 --- Definition current Sun Mar 11 00:51:23 2018 +-- Definition current Sat Jun 2 02:19:15 2018 ( SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE @@ -784,7 +790,7 @@ A B C >>invoke t031t1; -- Definition of Trafodion table TRAFODION.SCH.T031T1 --- Definition current Sun Mar 11 00:51:32 2018 +-- Definition current Sat Jun 2 02:19:22 2018 ( SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE @@ -804,7 +810,7 @@ A B C >>invoke t031t1; -- Definition of Trafodion table TRAFODION.SCH.T031T1 --- Definition current Sun Mar 11 00:51:37 2018 +-- Definition current Sat Jun 2 02:19:27 2018 ( SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE @@ -819,7 +825,7 @@ A B C >>invoke t031v1; -- Definition of Trafodion view TRAFODION.SCH.T031V1 --- Definition current Sun Mar 11 00:51:41 2018 +-- Definition current Sat Jun 2 02:19:30 2018 ( A INT DEFAULT NULL @@ -842,7 +848,7 @@ A B C >>invoke t031v1; -- Definition of Trafodion view TRAFODION.SCH.T031V1 --- Definition current Sun Mar 11 00:51:45 2018 +-- Definition current Sat Jun 2 02:19:35 2018 ( A INT DEFAULT NULL @@ -860,7 +866,7 @@ A B C >>invoke t031v1; -- Definition of Trafodion view TRAFODION.SCH.T031V1 --- Definition current Sun Mar 11 00:51:52 2018 +-- Definition current Sat Jun 2 02:19:40 2018 ( A INT DEFAULT NULL @@ -947,7 +953,7 @@ CREATE INDEX T031T1I1 ON TRAFODION.SCH.T031T1 >>invoke table(index_table t031t1i1); -- Definition of Trafodion table TRAFODION.SCH.T031T1I1 --- Definition current Sun Mar 11 00:52:20 2018 +-- Definition current Sat Jun 2 02:20:02 2018 ( "A@" INT NO DEFAULT @@ -1082,7 +1088,7 @@ HIVE >>showddl hive.hive.t031hivet1; /* Hive DDL */ -CREATE TABLE T031HIVET1 +CREATE TABLE HIVE.HIVE.T031HIVET1 ( A string ) @@ -1092,7 +1098,7 @@ CREATE TABLE T031HIVET1 /* Trafodion DDL */ REGISTER /*INTERNAL*/ HIVE TABLE HIVE.HIVE.T031HIVET1; -/* ObjectUID = 596882092547840046 */ +/* ObjectUID = 6094095443794522678 */ CREATE EXTERNAL TABLE T031HIVET1 FOR HIVE.HIVE.T031HIVET1 @@ -1259,8 +1265,8 @@ A A@ SYSKEY ----------- -------------------- - 1 2046478438118704386 - 2 2046478438119028079 + 1 6394711054759248971 + 2 6394711054759564196 --- 2 row(s) selected. >> http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/regress/seabase/TEST026 ---------------------------------------------------------------------- diff --git a/core/sql/regress/seabase/TEST026 b/core/sql/regress/seabase/TEST026 index b16ba51..6ef7d47 100644 --- a/core/sql/regress/seabase/TEST026 +++ b/core/sql/regress/seabase/TEST026 @@ -28,6 +28,7 @@ cleanup table t026t1; cleanup schema sch026; create schema sch026; cleanup metadata; +cleanup metadata, return details; log LOG026 clear; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/regress/seabase/TEST031 ---------------------------------------------------------------------- diff --git a/core/sql/regress/seabase/TEST031 b/core/sql/regress/seabase/TEST031 index aa9cf09..6d2b182 100644 --- a/core/sql/regress/seabase/TEST031 +++ b/core/sql/regress/seabase/TEST031 @@ -215,8 +215,11 @@ control query shape cut ; cqd hdfs_io_buffersize reset ; cqd hive_min_bytes_per_esp_partition reset; --- should return error. +-- should not return error. +process hive statement 'drop table t'; +process hive statement 'create table t (a int)'; process hive statement 'insert into t values (1)'; +process hive statement 'drop table t'; -- default USER drop table if exists t031t1; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/sqlcomp/CmpDescribe.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpDescribe.cpp b/core/sql/sqlcomp/CmpDescribe.cpp index 97f2014..420167e 100644 --- a/core/sql/sqlcomp/CmpDescribe.cpp +++ b/core/sql/sqlcomp/CmpDescribe.cpp @@ -181,38 +181,7 @@ static short CmpDescribeTransaction( char *&outbuf, ULng32 &outbuflen, NAMemory *h); - -short CmpDescribeHiveTable ( - const CorrName &dtName, - short type, // 1, invoke. 2, showddl. 3, createLike - char* &outbuf, - ULng32 &outbuflen, - CollHeap *heap, - UInt32 columnLengthLimit = UINT_MAX); - -short CmpDescribeSeabaseTable ( - const CorrName &dtName, - short type, // 1, invoke. 2, showddl. 3, createLike - char* &outbuf, - ULng32 &outbuflen, - CollHeap *heap, - const char * pkeyStr = NULL, - NABoolean withPartns = FALSE, - NABoolean withoutSalt = FALSE, - NABoolean withoutDivisioning = FALSE, - NABoolean withoutRowFormat = FALSE, - NABoolean withoutLobColumns = FALSE, - UInt32 columnLengthLimit = UINT_MAX, - NABoolean noTrailingSemi = FALSE, - - // used to add,rem,alter column definition from col list. - // valid for 'createLike' mode. - // Used for 'alter add/drop/alter col'. - char * colName = NULL, - short ada = 0, // 0,add. 1,drop. 2,alter - const NAColumn * nacol = NULL, - const NAType * natype = NULL, - Space *inSpace = NULL); + short CmpDescribeSequence ( const CorrName &dtName, @@ -871,7 +840,8 @@ short CmpDescribe(const char *query, const RelExpr *queryExpr, rc = CmpDescribeHiveTable(d->getDescribedTableName(), (d->getFormat() == Describe::INVOKE_ ? 1 : 2), - outbuf, outbuflen, heap); + outbuf, outbuflen, heap, + d->getIsDetail()); goto finally; // we are done } @@ -892,7 +862,10 @@ short CmpDescribe(const char *query, const RelExpr *queryExpr, rc = CmpDescribeSeabaseTable(d->getDescribedTableName(), (d->getFormat() == Describe::INVOKE_ ? 1 : 2), - outbuf, outbuflen, heap, NULL, TRUE); + outbuf, outbuflen, heap, NULL, TRUE, + FALSE, FALSE, FALSE, FALSE, UINT_MAX, FALSE, + NULL, 0, NULL, NULL, NULL, + d->getIsDetail()); goto finally; // we are done } @@ -2218,10 +2191,11 @@ short CmpDescribeHiveTable ( char* &outbuf, ULng32 &outbuflen, CollHeap *heap, + NABoolean isDetail, UInt32 columnLengthLimit) { const NAString& tableName = - dtName.getQualifiedNameObj().getObjectName(); + dtName.getQualifiedNameObj().getQualifiedNameAsString(); BindWA bindWA(ActiveSchemaDB(), CmpCommon::context(), FALSE/*inDDL*/); NATable *naTable = bindWA.getNATable((CorrName&)dtName); @@ -2331,8 +2305,13 @@ short CmpDescribeHiveTable ( { outputShortLine(space,"/* Hive DDL */"); - sprintf(buf, "CREATE TABLE %s", - tableName.data()); + if (naTable->isHiveExternalTable()) + sprintf(buf, "CREATE EXTERNAL TABLE %s", + tableName.data()); + else + sprintf(buf, "CREATE TABLE %s", + tableName.data()); + outputShortLine(space, buf); } @@ -2403,9 +2382,9 @@ short CmpDescribeHiveTable ( else if (hTabStats->isSequenceFile()) { if (type == 1) - outputShortLine(space, " /* stored as sequence */"); + outputShortLine(space, " /* stored as sequencefile */"); else if (type == 2) - outputShortLine(space, " stored as sequence "); + outputShortLine(space, " stored as sequencefile "); } } @@ -2418,23 +2397,38 @@ short CmpDescribeHiveTable ( } // if this hive table is registered in traf metadata, show that. - if ((type == 2) && - (naTable->isRegistered())) + if (type == 2) { - Int64 objectUID = (Int64)naTable->objectUid().get_value(); - - outputShortLine(space, " "); - - sprintf(buf, "REGISTER%sHIVE %s %s;", - (naTable->isInternalRegistered() ? " /*INTERNAL*/ " : " "), - (isView ? "VIEW" : "TABLE"), - naTable->getTableName().getQualifiedNameAsString().data()); - - NAString bufnas(buf); - outputLongLine(space, bufnas, 0); + if (naTable->isRegistered()) + { + Int64 objectUID = (Int64)naTable->objectUid().get_value(); + + outputShortLine(space, " "); + + sprintf(buf, "REGISTER%sHIVE %s %s;", + (naTable->isInternalRegistered() ? " /*INTERNAL*/ " : " "), + (isView ? "VIEW" : "TABLE"), + naTable->getTableName().getQualifiedNameAsString().data()); + + NAString bufnas(buf); + outputLongLine(space, bufnas, 0); + + str_sprintf(buf, "/* ObjectUID = %ld */", objectUID); + outputShortLine(space, buf); + } + else if (isDetail) + { + // show a comment that this object should be registered + outputShortLine(space, " "); - str_sprintf(buf, "/* ObjectUID = %ld */", objectUID); - outputShortLine(space, buf); + outputShortLine(space, "-- Object is not registered in Trafodion Metadata."); + outputShortLine(space, "-- Register it using the next command:"); + sprintf(buf, "-- REGISTER HIVE %s %s;", + (isView ? "VIEW" : "TABLE"), + naTable->getTableName().getQualifiedNameAsAnsiString().data()); + NAString bufnas(buf); + outputLongLine(space, bufnas, 0); + } } // if this hive table has an associated external table, show ddl @@ -2523,6 +2517,98 @@ short CmpDescribeHiveTable ( return 0; } +// this method is used to convert a trafodion table definition to corresponding +// Hive 'create' DDL. +// Used when a Hive table is being create 'like' a traf table. +short CmpDescribeTrafAsHiveTable ( + const CorrName &dtName, + short type, + char* &outbuf, + ULng32 &outbuflen, + CollHeap *heap, + UInt32 columnLengthLimit) +{ + BindWA bindWA(ActiveSchemaDB(), CmpCommon::context(), FALSE/*inDDL*/); + NATable *naTable = bindWA.getNATable((CorrName&)dtName); + if (naTable == NULL || bindWA.errStatus()) + return -1; + + if (NOT naTable->isSeabaseTable()) + return -1; + + char * buf = new (heap) char[15000]; + CMPASSERT(buf); + + time_t tp; + time(&tp); + + Space space; + + if (!CmpDescribeIsAuthorized(SQLOperation::UNKNOWN, + naTable->getPrivInfo(), + COM_BASE_TABLE_OBJECT)) + return -1; + + // emit an initial newline + outputShortLine(space, " "); + + outputShortLine(space, " ( "); + + Int32 ii = 0; + for (Int32 i = 0; i < (Int32)naTable->getColumnCount(); i++) + { + NAColumn * nac = naTable->getNAColumnArray()[i]; + + if (nac->isSystemColumn()) + continue; + + NAString colName = nac->getColName(); + colName.toLower(); + + const NAType * nat = nac->getType(); + + sprintf(buf, "%s ", colName.data()); + + NAString nas; + nat->getMyTypeAsHiveText(&nas); + + // if it is a character type and it is longer than the length + // limit in bytes, then shorten the target type + if ((nat->getTypeQualifier() == NA_CHARACTER_TYPE) && + (!nat->isLob()) && + (columnLengthLimit < UINT_MAX)) + { + const CharType * natc = (const CharType *)nat; + if (natc->getDataStorageSize() > columnLengthLimit) + { + CharType * newType = (CharType *)natc->newCopy(NULL); + newType->setDataStorageSize(columnLengthLimit); + nas.clear(); + newType->getMyTypeAsText(&nas, FALSE); + delete newType; + } + } + + sprintf(&buf[strlen(buf)], "%s", nas.data()); + + NAString colString(buf); + Int32 j = ii; + outputColumnLine(space, colString, j); + + ii++; + } + + outputShortLine(space, " )"); + + outbuflen = space.getAllocatedSpaceSize(); + outbuf = new (heap) char[outbuflen]; + space.makeContiguous(outbuf, outbuflen); + + NADELETEBASIC(buf, heap); + + return 0; +} + // type: 1, invoke. 2, showddl. 3, create_like short cmpDisplayColumn(const NAColumn *nac, char * inColName, @@ -2905,7 +2991,8 @@ short CmpDescribeSeabaseTable ( short ada, const NAColumn * nacol, const NAType * natype, - Space *inSpace) + Space *inSpace, + NABoolean isDetail) { const NAString& tableName = dtName.getQualifiedNameObj().getQualifiedNameAsAnsiString(TRUE); @@ -2943,8 +3030,8 @@ short CmpDescribeSeabaseTable ( } NABoolean isVolatile = naTable->isVolatileTable(); - NABoolean isExternalTable = naTable->isExternalTable(); - NABoolean isImplicitExternalTable = naTable->isImplicitExternalTable(); + NABoolean isExternalTable = naTable->isTrafExternalTable(); + NABoolean isImplicitExternalTable = naTable->isImplicitTrafExternalTable(); NABoolean isHbaseMapTable = naTable->isHbaseMapTable(); NABoolean isHbaseCellOrRowTable = (naTable->isHbaseCellTable() || naTable->isHbaseRowTable()); @@ -3844,6 +3931,18 @@ short CmpDescribeSeabaseTable ( str_sprintf(buf, "/* ObjectUID = %ld */", objectUID); outputShortLine(*space, buf); } + else if (isDetail) + { + // show a comment that this object should be registered + outputShortLine(*space, " "); + + outputShortLine(*space, "-- Object is not registered in Trafodion Metadata."); + outputShortLine(*space, "-- Register it using the next command:"); + sprintf(buf, "-- REGISTER HBASE TABLE %s;", + naTable->getTableName().getQualifiedNameAsAnsiString().data()); + NAString bufnas(buf); + outputLongLine(*space, bufnas, 0); + } } //display comments http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/sqlcomp/CmpDescribe.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpDescribe.h b/core/sql/sqlcomp/CmpDescribe.h index 7a9004f..01d11a4 100644 --- a/core/sql/sqlcomp/CmpDescribe.h +++ b/core/sql/sqlcomp/CmpDescribe.h @@ -43,8 +43,6 @@ class ExeCliInterface; short exeImmedOneStmt(const char *stmt); -//short exeImmedOneStmt(void *in_sql_src, -// const char *stmt); short sendAllControls(NABoolean copyCQS, NABoolean sendAllCQDs, @@ -58,4 +56,69 @@ void sendParserFlag (ULng32 flag); short setParentQidAtSession(NAHeap *heap, const char *parentQid); +extern short CmpDescribeSeabaseTable ( + const CorrName &dtName, + short type, // 1, invoke. 2, showddl. 3, createLike + char* &outbuf, + ULng32 &outbuflen, + CollHeap *heap, + const char * pkeyStr = NULL, + NABoolean withPartns = FALSE, + NABoolean withoutSalt = FALSE, + NABoolean withoutDivisioning = FALSE, + NABoolean withoutRowFormat = FALSE, + NABoolean withoutLobColumns = FALSE, + UInt32 columnLengthLimit = UINT_MAX, + NABoolean noTrailingSemi = FALSE, + + // used to add,rem,alter column definition from col list. + // valid for 'createLike' mode. + // Used for 'alter add/drop/alter col'. + char * colName = NULL, + short ada = 0, // 0,add. 1,drop. 2,alter + const NAColumn * nacol = NULL, + const NAType * natype = NULL, + Space *inSpace = NULL, + NABoolean isDetail = FALSE); + +short CmpDescribeHiveTable ( + const CorrName &dtName, + short type, // 1, invoke. 2, showddl. 3, createLike + char* &outbuf, + ULng32 &outbuflen, + CollHeap *heap, + NABoolean isDetail = FALSE, + UInt32 columnLengthLimit = UINT_MAX); + +short CmpDescribeTrafAsHiveTable ( + const CorrName &dtName, + short type, // 1, invoke. 2, showddl. 3, createLike + char* &outbuf, + ULng32 &outbuflen, + CollHeap *heap, + UInt32 columnLengthLimit = UINT_MAX); + +// type: 1, invoke. 2, showddl. 3, create_like +extern short cmpDisplayColumn(const NAColumn *nac, + char * inColName, + const NAType *inNAT, + short displayType, + Space *inSpace, + char * buf, + Lng32 &ii, + NABoolean namesOnly, + NABoolean &identityCol, + NABoolean isExternalTable, + NABoolean isAlignedRowFormat, + UInt32 columnLengthLimit, + NAList<const NAColumn *> * truncatedColumnList); + +extern short cmpDisplayPrimaryKey(const NAColumnArray & naColArr, + Lng32 numKeys, + NABoolean displaySystemCols, + Space &space, char * buf, + NABoolean displayCompact, + NABoolean displayAscDesc, + NABoolean displayParens); + #endif http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/sqlcomp/CmpSeabaseDDL.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpSeabaseDDL.h b/core/sql/sqlcomp/CmpSeabaseDDL.h index a08a345..9133f34 100644 --- a/core/sql/sqlcomp/CmpSeabaseDDL.h +++ b/core/sql/sqlcomp/CmpSeabaseDDL.h @@ -45,6 +45,7 @@ #include "PrivMgrMD.h" #include "ElemDDLHbaseOptions.h" #include "CmpContext.h" +#include "parser.h" class ExpHbaseInterface; class ExeCliInterface; @@ -1085,6 +1086,22 @@ protected: const ComObjectName &tgtTableName, const ComObjectName &srcTableName); + static short genDDLforHiveTableLikeTrafTable( + StmtDDLCreateTable * createTableNode, + NAString &currCatName, NAString &currSchName, + NAString &tableDDL); // output. Contains hive DDL string. + +public: + static NABoolean setupQueryTreeForHiveDDL( + Parser::HiveDDLInfo * hiveDDLInfo, + char * inputStr, + CharInfo::CharSet inputStrCharSet, + NAString currCatName, + NAString currSchName, + ExprNode** node); + +protected: + // makes a copy of underlying hbase table short cloneHbaseTable( const NAString &srcTable, const NAString &clonedTable, @@ -1368,6 +1385,14 @@ protected: NABoolean cascade ); + short unregisterHiveSchema + ( + const NAString &catalogNamePart, + const NAString &schemaNamePart, + ExeCliInterface &cliInterface, + NABoolean cascade + ); + void regOrUnregNativeObject ( StmtDDLRegOrUnregObject * regOrUnregObject, NAString &currCatName, NAString &currSchName); @@ -1415,6 +1440,9 @@ protected: StmtDDLDropHbaseTable * dropTableNode, NAString &currCatName, NAString &currSchName); + void processDDLonHiveObjects(StmtDDLonHiveObjects * hddl, + NAString &currCatName, NAString &currSchName); + void initSeabaseMD(NABoolean ddlXns, NABoolean minimal); void dropSeabaseMD(NABoolean ddlXns); void createSeabaseMDviews(); http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp b/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp index 49105c2..f08b642 100644 --- a/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp +++ b/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp @@ -66,7 +66,8 @@ CmpSeabaseMDcleanup::CmpSeabaseMDcleanup(NAHeap *heap) numOrphanHbaseEntries_(0), numOrphanObjectsEntries_(0), numOrphanViewsEntries_(0), - numInconsistentHiveEntries_(0) + numInconsistentHiveEntries_(0), + isHive_(FALSE) {}; Int64 CmpSeabaseMDcleanup::getCleanupObjectUID( @@ -298,16 +299,29 @@ short CmpSeabaseMDcleanup::validateInputValues( objectOwner_ = -1; } + isHive_ = FALSE; + if (catName_ == HIVE_SYSTEM_CATALOG) + isHive_ = TRUE; + // generate hbase name that will be used to drop underlying hbase object extNameForHbase_ = ""; - if ((objType_ == COM_BASE_TABLE_OBJECT_LIT) || - (objType_ == COM_INDEX_OBJECT_LIT)) + if ((NOT isHive_) && + ((objType_ == COM_BASE_TABLE_OBJECT_LIT) || + (objType_ == COM_INDEX_OBJECT_LIT))) { if (NOT (catName_.isNull() || schName_.isNull() || objName_.isNull())) { extNameForHbase_ = catName_ + "." + schName_ + "." + objName_; } } + else if (isHive_) + { + if (NOT ((schName_.compareTo(HIVE_DEFAULT_SCHEMA_EXE, NAString::ignoreCase) == 0) || + (schName_.compareTo(HIVE_SYSTEM_SCHEMA, NAString::ignoreCase) == 0))) + extNameForHive_ = schName_ + "."; + + extNameForHive_ += objName_; + } // Make sure user has necessary privileges to perform drop if (!isDDLOperationAuthorized(SQLOperation::DROP_TABLE, @@ -475,6 +489,34 @@ short CmpSeabaseMDcleanup::gatherDependentObjects(ExeCliInterface *cliInterface) } } + if (isHive_) + { + // if this hive table has an external table, get its uid + NAString extTableName; + extTableName = ComConvertNativeNameToTrafName(catName_, schName_, objName_); + if (NOT extTableName.isNull()) + { + QualifiedName qn(extTableName, 3); + Int64 extObjUID = + getObjectUID(cliInterface, + qn.getCatalogName(), qn.getSchemaName(), qn.getObjectName(), + COM_BASE_TABLE_OBJECT_LIT, + NULL, NULL, FALSE, + FALSE); + if (extObjUID > 0) + { + char query[1000]; + str_sprintf(query, "cleanup uid %ld", extObjUID); + cliRC = cliInterface->executeImmediate(query); + if (cliRC < 0) + { + cliInterface->retrieveSQLDiagnostics(CmpCommon::diags()); + return -1; + } + } + } + } + if (errorSeen) return -1; else @@ -733,8 +775,19 @@ short CmpSeabaseMDcleanup::deleteHistogramEntries(ExeCliInterface *cliInterface) (objUID_ > 0) && (NOT catName_.isNull()) && (NOT schName_.isNull())) - if (dropSeabaseStats(cliInterface, catName_.data(), schName_.data(), objUID_)) - return -1; + { + if (NOT isHive_) + { + if (dropSeabaseStats(cliInterface, catName_.data(), schName_.data(), objUID_)) + return -1; + } + else + { + if (dropSeabaseStats(cliInterface, HIVE_STATS_CATALOG, + HIVE_STATS_SCHEMA_NO_QUOTES, objUID_)) + return -1; + } + } return 0; } @@ -1035,81 +1088,6 @@ short CmpSeabaseMDcleanup::cleanupUIDs(ExeCliInterface *cliInterface, return 0; } -void CmpSeabaseMDcleanup::cleanupHiveObject(const StmtDDLCleanupObjects * stmtCleanupNode, - ExeCliInterface *cliInterface) -{ - - Lng32 cliRC = 0; - char query[1000]; - NABoolean errorSeen = FALSE; - - // check if this table exists in hive metadata - NABoolean hiveObjExists = TRUE; - NAString objName(stmtCleanupNode->getTableNameAsQualifiedName()->getObjectName()); - objName.toLower(); - str_sprintf(query, "select * from (get %s in schema %s.%s, no header, match '%s') x(a)", - (stmtCleanupNode->getType() == StmtDDLCleanupObjects::HIVE_TABLE_) - ? "tables" : "views", - stmtCleanupNode->getTableNameAsQualifiedName()->getCatalogName().data(), - stmtCleanupNode->getTableNameAsQualifiedName()->getSchemaName().data(), - objName.data()); - cliRC = cliInterface->fetchRowsPrologue(query, TRUE/*no exec*/); - if (cliRC < 0) - { - cliInterface->retrieveSQLDiagnostics(CmpCommon::diags()); - return; - } - - cliRC = cliInterface->clearExecFetchClose(NULL, 0); - if (cliRC < 0) - { - cliInterface->retrieveSQLDiagnostics(CmpCommon::diags()); - return; - } - - if (cliRC == 100) // did not find the row - { - hiveObjExists = FALSE; - } - - // if underlying hive object doesn't exist, drop external table and unregister - // objects - if (NOT hiveObjExists) - { - // drop external table - if (stmtCleanupNode->getType() == StmtDDLCleanupObjects::HIVE_TABLE_) - { - str_sprintf(query, "drop external table if exists %s for %s;", - objName.data(), - stmtCleanupNode->getTableNameAsQualifiedName()-> - getQualifiedNameAsString().data()); - cliRC = cliInterface->executeImmediate(query); - if (cliRC < 0) - { - if (processCleanupErrors(NULL, errorSeen)) - return; - } - } - - // unregister registered table or view - if (stmtCleanupNode->getType() == StmtDDLCleanupObjects::HIVE_TABLE_) - str_sprintf(query, "unregister hive table if exists %s cleanup;", - stmtCleanupNode->getTableNameAsQualifiedName()->getQualifiedNameAsString().data()); - else - str_sprintf(query, "unregister hive view if exists %s cleanup;", - stmtCleanupNode->getTableNameAsQualifiedName()->getQualifiedNameAsString().data()); - - cliRC = cliInterface->executeImmediate(query); - if (cliRC < 0) - { - if (processCleanupErrors(NULL, errorSeen)) - return; - } - } - - return; -} - void CmpSeabaseMDcleanup::cleanupHBaseObject(const StmtDDLCleanupObjects * stmtCleanupNode, ExeCliInterface *cliInterface) { @@ -1908,6 +1886,16 @@ void CmpSeabaseMDcleanup::cleanupMetadataEntries(ExeCliInterface *cliInterface, dws->setBlackBoxLen(blackBoxLen); dws->setBlackBox(blackBox); + if ((numOrphanViewsEntries_ == 0) && + (blackBoxLen > 0)) + { + str_sprintf(buf, " End: Cleanup Inconsistent Views Entries (%d %s %s) [internal error: blackBoxLen = %d] ", + numOrphanViewsEntries_, + (numOrphanViewsEntries_ == 1 ? "entry" : "entries"), + (checkOnly_ ? "found" : "cleaned up"), + blackBoxLen); + } + dws->setMsg(buf); dws->setStep(HIVE_ENTRIES); dws->setSubstep(0); @@ -2036,13 +2024,6 @@ void CmpSeabaseMDcleanup::cleanupObjects(StmtDDLCleanupObjects * stmtCleanupNode } if (stmtCleanupNode && - ((stmtCleanupNode->getType() == StmtDDLCleanupObjects::HIVE_TABLE_) || - (stmtCleanupNode->getType() == StmtDDLCleanupObjects::HIVE_VIEW_))) - { - return cleanupHiveObject(stmtCleanupNode, &cliInterface); - } - - if (stmtCleanupNode && (stmtCleanupNode->getType() == StmtDDLCleanupObjects::HBASE_TABLE_)) { return cleanupHBaseObject(stmtCleanupNode, &cliInterface); @@ -2056,6 +2037,7 @@ void CmpSeabaseMDcleanup::cleanupObjects(StmtDDLCleanupObjects * stmtCleanupNode if (((objType_ == COM_BASE_TABLE_OBJECT_LIT) || (objType_ == COM_INDEX_OBJECT_LIT)) && + (NOT isHive_) && (extNameForHbase_.isNull())) { // add warning that name couldnt be found. Hbase object cannot be removed. @@ -2091,7 +2073,8 @@ void CmpSeabaseMDcleanup::cleanupObjects(StmtDDLCleanupObjects * stmtCleanupNode if (stopOnError_) goto label_error; - if (NOT extNameForHbase_.isNull()) + if ((NOT isHive_) && + (NOT extNameForHbase_.isNull())) { HbaseStr hbaseObject; hbaseObject.val = (char*)extNameForHbase_.data(); @@ -2104,6 +2087,29 @@ void CmpSeabaseMDcleanup::cleanupObjects(StmtDDLCleanupObjects * stmtCleanupNode goto label_return; } + // drop underlying Hive object + if (isHive_) + { + NAString hiveQuery; + if (objType_ == COM_BASE_TABLE_OBJECT_LIT) + { + hiveQuery = "drop table if exists " + extNameForHive_; + } + else if (objType_ == COM_VIEW_OBJECT_LIT) + { + hiveQuery = "drop view if exists " + extNameForHive_; + } + + if (NOT hiveQuery.isNull()) + { + if (HiveClient_JNI::executeHiveSQL(hiveQuery.data()) != HVC_OK) + { + if (stopOnError_) + goto label_return; + } + } + } + cliRC = dropIndexes(&cliInterface); if (cliRC) if (stopOnError_) @@ -2138,7 +2144,8 @@ void CmpSeabaseMDcleanup::cleanupObjects(StmtDDLCleanupObjects * stmtCleanupNode ( cn, ComQiScope::REMOVE_FROM_ALL_USERS, - COM_BASE_TABLE_OBJECT, + (objType_ == COM_VIEW_OBJECT_LIT ? COM_VIEW_OBJECT : + COM_BASE_TABLE_OBJECT), FALSE, FALSE); } http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/sqlcomp/CmpSeabaseDDLcleanup.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpSeabaseDDLcleanup.h b/core/sql/sqlcomp/CmpSeabaseDDLcleanup.h index 29a3b3a..5132fb2 100644 --- a/core/sql/sqlcomp/CmpSeabaseDDLcleanup.h +++ b/core/sql/sqlcomp/CmpSeabaseDDLcleanup.h @@ -119,9 +119,6 @@ class CmpSeabaseMDcleanup : public CmpSeabaseDDL ExeCliInterface *cliInterface, ExpHbaseInterface *ehi); - void cleanupHiveObject(const StmtDDLCleanupObjects * stmtCleanupNode, - ExeCliInterface *cliInterface); - void cleanupHBaseObject(const StmtDDLCleanupObjects * stmtCleanupNode, ExeCliInterface *cliInterface); @@ -153,10 +150,12 @@ class CmpSeabaseMDcleanup : public CmpSeabaseDDL // stop cleanup if an error occurs NABoolean stopOnError_; + NABoolean isHive_; NAString catName_; NAString schName_; NAString objName_; NAString extNameForHbase_; + NAString extNameForHive_; NAString objType_; // BT, IX, SG... Int64 objUID_; Int32 objectOwner_; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp b/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp index b97feba..35869f9 100644 --- a/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp +++ b/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp @@ -769,6 +769,10 @@ short CmpSeabaseDDL::createMDdescs(MDDescsInfo *&trafMDDescsInfo) // Load definitions of system metadata tables for (size_t i = 0; i < numMDTables; i++) { + // no need to do hive ddl checks for MD query compiles + parser.hiveDDLInfo_->init(); + parser.hiveDDLInfo_->disableDDLcheck_ = TRUE; + const MDTableInfo &mdti = allMDtablesInfo[i]; const char * oldName = NULL; @@ -9073,8 +9077,8 @@ short CmpSeabaseDDL::executeSeabaseDDL(DDLExpr * ddlExpr, ExprNode * ddlNode, ddlNode->castToStmtDDLNode()->castToStmtDDLCreateTable(); if ((createTableParseNode->getAddConstraintUniqueArray().entries() > 0) || - (createTableParseNode->getAddConstraintRIArray().entries() > 0) || - (createTableParseNode->getAddConstraintCheckArray().entries() > 0)) + (createTableParseNode->getAddConstraintRIArray().entries() > 0) || + (createTableParseNode->getAddConstraintCheckArray().entries() > 0)) createSeabaseTableCompound(createTableParseNode, currCatName, currSchName); else { @@ -9549,6 +9553,13 @@ short CmpSeabaseDDL::executeSeabaseDDL(DDLExpr * ddlExpr, ExprNode * ddlNode, doSeabaseCommentOn(comment, currCatName, currSchName); } + else if (ddlNode->getOperatorType() == DDL_ON_HIVE_OBJECTS) + { + StmtDDLonHiveObjects * hddl = + ddlNode->castToStmtDDLNode()->castToStmtDDLonHiveObjects(); + + processDDLonHiveObjects(hddl, currCatName, currSchName); + } else { // some operator type that this routine doesn't support yet http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/sqlcomp/CmpSeabaseDDLschema.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpSeabaseDDLschema.cpp b/core/sql/sqlcomp/CmpSeabaseDDLschema.cpp index e346c9e..c1e57f1 100644 --- a/core/sql/sqlcomp/CmpSeabaseDDLschema.cpp +++ b/core/sql/sqlcomp/CmpSeabaseDDLschema.cpp @@ -371,9 +371,8 @@ ComObjectType objectType; output = "/* Hive DDL */"; outlines.push_back(output.data()); - output = "create database "; + output = "CREATE SCHEMA HIVE."; NAString lsch(schemaName); - lsch.toLower(); output += lsch.data(); output += ";"; @@ -383,7 +382,7 @@ ComObjectType objectType; if (isHiveRegistered) { - output = "REGISTER /*INTERNAL*/ HIVE SCHEMA hive."; + output = "REGISTER /*INTERNAL*/ HIVE SCHEMA HIVE."; output += lsch.data(); output += ";";
