TRAFODION-3086 Traf support for DDL operations on Hive objects -- Support for TRAFODION-3086. Details in document attached to jira.
Other changes: -- support for "if not exists", "if exists" clause for create/drop view -- Support for: truncate T, truncate table T. -- same as purgedata -- showddl <tab>, detail -- unregister hive schema <sch> -- will unregister all objects in specified schema -- Support for "if not registered", "if registered" clauss for register/unregister command. Project: http://git-wip-us.apache.org/repos/asf/trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/trafodion/commit/6f490daf Tree: http://git-wip-us.apache.org/repos/asf/trafodion/tree/6f490daf Diff: http://git-wip-us.apache.org/repos/asf/trafodion/diff/6f490daf Branch: refs/heads/master Commit: 6f490daf1b30e6ced48ce43bbd067a0bd828e620 Parents: 8dbf5df Author: Anoop Sharma <[email protected]> Authored: Mon Jun 4 17:33:12 2018 +0000 Committer: Anoop Sharma <[email protected]> Committed: Mon Jun 4 17:33:12 2018 +0000 ---------------------------------------------------------------------- core/sqf/sql/scripts/install_local_hadoop | 22 +- core/sql/bin/SqlciErrors.txt | 10 +- core/sql/comexe/ComTdbExeUtil.cpp | 15 - core/sql/comexe/ComTdbExeUtil.h | 14 - core/sql/common/ComObjectName.cpp | 7 +- core/sql/common/NAString.cpp | 27 +- core/sql/common/NAType.cpp | 10 +- core/sql/common/OperTypeEnum.h | 4 + core/sql/executor/ExComTdb.cpp | 1 - core/sql/executor/ExExeUtil.h | 54 - core/sql/executor/ExExeUtilMisc.cpp | 39 - core/sql/executor/HiveClient_JNI.cpp | 3 +- core/sql/executor/JavaObjectInterface.cpp | 15 +- core/sql/executor/JavaObjectInterface.h | 7 +- core/sql/export/ComDiags.cpp | 1 + core/sql/generator/GenExplain.cpp | 25 +- core/sql/generator/GenFastTransport.cpp | 6 + core/sql/generator/GenPreCode.cpp | 3 +- core/sql/generator/GenRelExeUtil.cpp | 20 - core/sql/generator/GenRelScan.cpp | 4 + core/sql/optimizer/BindRelExpr.cpp | 5 +- core/sql/optimizer/NATable.cpp | 26 +- core/sql/optimizer/NATable.h | 41 +- core/sql/optimizer/RelExeUtil.cpp | 91 +- core/sql/optimizer/RelExeUtil.h | 11 +- core/sql/optimizer/RelExpr.cpp | 4 +- core/sql/optimizer/hiveHook.h | 16 + core/sql/parser/AllStmtDDL.h | 2 + core/sql/parser/ElemDDLLikeOptions.cpp | 30 + core/sql/parser/ElemDDLLikeOptions.h | 33 + core/sql/parser/ElemDDLNode.cpp | 6 + core/sql/parser/ElemDDLNode.h | 6 +- core/sql/parser/ParDDLLikeOpts.cpp | 15 + core/sql/parser/ParDDLLikeOptsCreateTable.h | 14 + core/sql/parser/SqlParserAux.cpp | 75 +- core/sql/parser/SqlParserAux.h | 14 +- core/sql/parser/StmtDDLCleanupObjects.h | 4 +- core/sql/parser/StmtDDLCreate.cpp | 7 +- core/sql/parser/StmtDDLCreateTable.h | 14 + core/sql/parser/StmtDDLCreateView.h | 7 + core/sql/parser/StmtDDLDrop.cpp | 3 +- core/sql/parser/StmtDDLDropView.h | 6 + core/sql/parser/StmtDDLMisc.cpp | 2 - core/sql/parser/StmtDDLRegOrUnregHive.cpp | 4 +- core/sql/parser/StmtDDLRegOrUnregHive.h | 6 +- core/sql/parser/StmtDDLonHiveObjects.h | 191 +++ core/sql/parser/sqlparser.y | 761 ++++++++---- core/sql/parser/ulexer.cpp | 25 +- core/sql/regress/compGeneral/EXPECTEDTOK | 4 +- core/sql/regress/compGeneral/EXPECTEDTOK2 | 4 +- core/sql/regress/hive/DIFF003.KNOWN | 10 +- core/sql/regress/hive/DIFF008.KNOWN | 7 + core/sql/regress/hive/DIFF009.KNOWN | 3 + core/sql/regress/hive/EXPECTED003 | 2 +- core/sql/regress/hive/EXPECTED005 | 187 ++- core/sql/regress/hive/EXPECTED007 | 314 ++++- core/sql/regress/hive/EXPECTED008 | 1105 ++++++++++++++++++ core/sql/regress/hive/EXPECTED009 | 96 +- core/sql/regress/hive/EXPECTED018 | 820 ++++++++----- core/sql/regress/hive/FILTER005 | 1 + core/sql/regress/hive/FILTER008 | 35 + core/sql/regress/hive/TEST003 | 2 +- core/sql/regress/hive/TEST005 | 55 +- core/sql/regress/hive/TEST005_a.hive.sql | 2 + core/sql/regress/hive/TEST007 | 53 +- core/sql/regress/hive/TEST008 | 304 +++++ core/sql/regress/hive/TEST009 | 4 +- core/sql/regress/hive/TEST009_a.hive.sql | 8 +- core/sql/regress/hive/TEST009_b.hive.sql | 26 +- core/sql/regress/hive/TEST018 | 154 ++- .../hive/TEST018_create_hive_tables.hive | 20 +- core/sql/regress/privs2/EXPECTED129 | 36 +- core/sql/regress/seabase/EXPECTED003 | 6 +- core/sql/regress/seabase/EXPECTED031 | 44 +- core/sql/regress/seabase/TEST026 | 1 + core/sql/regress/seabase/TEST031 | 5 +- core/sql/sqlcomp/CmpDescribe.cpp | 213 +++- core/sql/sqlcomp/CmpDescribe.h | 67 +- core/sql/sqlcomp/CmpSeabaseDDL.h | 28 + core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp | 185 +-- core/sql/sqlcomp/CmpSeabaseDDLcleanup.h | 5 +- core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp | 15 +- core/sql/sqlcomp/CmpSeabaseDDLschema.cpp | 5 +- core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 947 +++++++++++++-- core/sql/sqlcomp/CmpSeabaseDDLview.cpp | 20 + core/sql/sqlcomp/DefaultConstants.h | 3 + core/sql/sqlcomp/nadefaults.cpp | 4 +- core/sql/sqlcomp/parser.cpp | 131 ++- core/sql/sqlcomp/parser.h | 100 ++ .../main/java/org/trafodion/sql/HiveClient.java | 6 +- core/sql/ustat/hs_la.cpp | 6 +- 91 files changed, 5355 insertions(+), 1394 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sqf/sql/scripts/install_local_hadoop ---------------------------------------------------------------------- diff --git a/core/sqf/sql/scripts/install_local_hadoop b/core/sqf/sql/scripts/install_local_hadoop index 8f338b0..56ff179 100755 --- a/core/sqf/sql/scripts/install_local_hadoop +++ b/core/sqf/sql/scripts/install_local_hadoop @@ -726,21 +726,27 @@ else if [ $? -ne 0 ]; then (${JAVA_EXE} -version 2>&1) | grep '1.7' >/dev/null if [ $? -ne 0 ]; then - echo '**** ERROR:' - cat <<EOF - Please make sure you are using the Java 1.6 or 1.7 SDK. - Otherwise, download it into ${MY_SW_ROOT}, extract it and - make a symbolic link ${MY_SW_ROOT}/java that points to it and - export JAVA_HOME=${MY_SW_ROOT}/java + (${JAVA_EXE} -version 2>&1) | grep '1.8' >/dev/null + if [ $? -ne 0 ]; then + echo '**** ERROR:' + cat <<EOF + Please make sure you are using the Java 1.6 or 1.7 SDK. + Otherwise, download it into ${MY_SW_ROOT}, extract it and + make a symbolic link ${MY_SW_ROOT}/java that points to it and + export JAVA_HOME=${MY_SW_ROOT}/java EOF - echo exiting... - exit 1 + echo exiting... + exit 1 + else + echo "Java version 1.8 is ok" + fi else echo "Java version 1.7 is ok" fi else echo "Java version 1.6 is ok" fi + # end of check Java version echo http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/bin/SqlciErrors.txt ---------------------------------------------------------------------- diff --git a/core/sql/bin/SqlciErrors.txt b/core/sql/bin/SqlciErrors.txt index 23279fb..c493d86 100644 --- a/core/sql/bin/SqlciErrors.txt +++ b/core/sql/bin/SqlciErrors.txt @@ -213,7 +213,7 @@ 1211 ZZZZZ 99999 UUUUUUUU UUUUU UUUUUUU The column values of the SPLIT BY first key in position $0~Int0 (one-based) are not greater (for ascending columns) or less (for descending columns) than its predecessor. 1212 ZZZZZ 99999 UUUUUUUU UUUUU UUUUUUU An HBase region start key provided only part of the value for Trafodion key column $0~Int0. This could in some unusual situations cause errors. 1213 ZZZZZ 99999 UUUUUUUU UUUUU UUUUUUU Unable to create or initialize a connection to Apache Hive. -1214 ZZZZZ 99999 UUUUUUUU UUUUU UUUUUUU Error $0~String0 encountered when executing HiveQL statement $1~String1. +1214 ZZZZZ 99999 UUUUUUUU UUUUU UUUUUUU Error encountered when executing HiveQL statement "$1~String1". Details: $0~String0 1215 ZZZZZ 99999 UUUUUUUU UUUUU UUUUUUU An error occurred while determining host, port, or file name for HDFS URI $0~string0. Cause: $1~string1. 1220 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Code must contain two non-blank characters. 1221 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Only system components may contain system operations. @@ -382,8 +382,8 @@ 1384 ZZZZZ 99999 BEGINNER MINOR DBADMIN --- unused --- 1385 ZZZZZ 99999 BEGINNER MAJOR DBADMIN --- unused --- 1386 ZZZZZ 99999 BEGINNER MINOR DBADMIN --- unused --- -1387 ZZZZZ 99999 BEGINNER MINOR DBADMIN --- unused --- -1388 ZZZZZ 99999 BEGINNER MINOR DBADMIN Object $0~TableName does not exist in hive metadata. +1387 ZZZZZ 99999 BEGINNER MINOR DBADMIN $0~String0 $1~String1 already exists in Hive Metadata. +1388 ZZZZZ 99999 BEGINNER MINOR DBADMIN $0~String0 $1~String1 does not exist in Hive Metadata. 1389 ZZZZZ 99999 BEGINNER MINOR DBADMIN Object $0~String0 does not exist in Trafodion. 1390 ZZZZZ 99999 BEGINNER MINOR DBADMIN Object $0~String0 already exists in Trafodion. 1391 ZZZZZ 99999 BEGINNER MINOR DBADMIN $0~String0 is a $1~String1 object and cannot be updated. @@ -1309,8 +1309,8 @@ $1~String1 -------------------------------- 4259 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Primary Key clause must be specified for an HBase mapped table. 4260 ZZZZZ 99999 BEGINNER MAJOR DBADMIN HBase table($0~String0) to be mapped as an external table does not exist. 4261 ZZZZZ 99999 BEGINNER MAJOR DBADMIN The schema name $0~SchemaName is reserved for HBase mapped tables and cannot be explicitly specified. -4262 ZZZZZ 99999 BEGINNER MINOR DBADMIN Object $0~String0 has invalid state and cannot be accessed. It has an external table but the correponding hive table does not exist. Use 'cleanup hive table' command to cleanup this table. -4263 ZZZZZ 99999 BEGINNER MINOR DBADMIN Object $0~String0 has invalid state and cannot be accessed. It is registered in trafodion metadata but the correponding hive object does not exist. Use 'cleanup hive table/view' command to cleanup this object. +4262 ZZZZZ 99999 BEGINNER MINOR DBADMIN Object $0~String0 has invalid state and cannot be accessed. It has an external table but the correponding hive table does not exist. Use 'cleanup table' command to cleanup this table. +4263 ZZZZZ 99999 BEGINNER MINOR DBADMIN Object $0~String0 has invalid state and cannot be accessed. It is registered in trafodion metadata but the correponding hive object does not exist. Use 'cleanup table/view' command to cleanup this object. 4300 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Use of procedure $0~TableName was not valid - a UDR was invoked within a trigger. 4301 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Use of routine $0~TableName was not valid - return of result sets is not allowed within a trigger. 4302 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Procedure $0~TableName expects $1~Int0 parameters but was called with $1~int1 parameters. http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/comexe/ComTdbExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbExeUtil.cpp b/core/sql/comexe/ComTdbExeUtil.cpp index adb0390..c7c5a8c 100644 --- a/core/sql/comexe/ComTdbExeUtil.cpp +++ b/core/sql/comexe/ComTdbExeUtil.cpp @@ -1400,9 +1400,6 @@ void ComTdbExeUtilHiveTruncate::displayContents(Space * space, /////////////////////////////////////////////////////////////////////////// // -// Methods for class ComTdbExeUtilHiveQuery -// -/////////////////////////////////////////////////////////////////////////// ComTdbExeUtilHiveQuery::ComTdbExeUtilHiveQuery( char * hiveQuery, ULng32 hiveQueryLen, @@ -1428,34 +1425,27 @@ ComTdbExeUtilHiveQuery::ComTdbExeUtilHiveQuery( { setNodeType(ComTdb::ex_HIVE_QUERY); } - Long ComTdbExeUtilHiveQuery::pack(void * space) { if (hiveQuery_) hiveQuery_.pack(space); - return ComTdbExeUtil::pack(space); } - Lng32 ComTdbExeUtilHiveQuery::unpack(void * base, void * reallocator) { if(hiveQuery_.unpack(base)) return -1; - return ComTdbExeUtil::unpack(base, reallocator); } - void ComTdbExeUtilHiveQuery::displayContents(Space * space, ULng32 flag) { ComTdb::displayContents(space,flag & 0xFFFFFFFE); - if(flag & 0x00000008) { char buf[500]; str_sprintf(buf, "\nFor ComTdbExeUtilHiveQuery :"); space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(short)); - if (getHiveQuery() != NULL) { str_sprintf(buf,"HiveQuery = %s ",getHiveQuery()); @@ -1463,17 +1453,12 @@ void ComTdbExeUtilHiveQuery::displayContents(Space * space, sizeof(short)); } } - if (flag & 0x00000001) { displayExpression(space,flag); displayChildren(space,flag); } - } - -/////////////////////////////////////////////////////////////////////////// -// // Methods for class ComTdbExeUtilGetStatistics // /////////////////////////////////////////////////////////////////////////// http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/comexe/ComTdbExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbExeUtil.h b/core/sql/comexe/ComTdbExeUtil.h index ef48954..01d3519 100644 --- a/core/sql/comexe/ComTdbExeUtil.h +++ b/core/sql/comexe/ComTdbExeUtil.h @@ -1730,7 +1730,6 @@ public: ComTdbExeUtilHiveQuery() : ComTdbExeUtil() {} - ComTdbExeUtilHiveQuery(char * hiveQuery, ULng32 hiveQueryLen, ex_cri_desc * given_cri_desc, @@ -1740,36 +1739,23 @@ public: Lng32 num_buffers, ULng32 buffer_size ); - Long pack(void *); Lng32 unpack(void *, void * reallocator); - - // --------------------------------------------------------------------- - // Redefine virtual functions required for Versioning. - //---------------------------------------------------------------------- virtual short getClassSize() {return (short)sizeof(ComTdbExeUtilHiveQuery);} - virtual const char *getNodeName() const { return "HIVE_QUERY"; }; - char * getHiveQuery() const { return hiveQuery_; } - - // --------------------------------------------------------------------- - // Used by the internal SHOWPLAN command to get attributes of a TDB. - // --------------------------------------------------------------------- void displayContents(Space *space, ULng32 flag); - private: NABasicPtr hiveQuery_; // 00-07 UInt32 hiveQueryLen_; // 08-11 UInt32 flags_; // 12-15 }; - class ComTdbExeUtilGetStatistics : public ComTdbExeUtil { friend class ExExeUtilGetStatisticsTcb; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/common/ComObjectName.cpp ---------------------------------------------------------------------- diff --git a/core/sql/common/ComObjectName.cpp b/core/sql/common/ComObjectName.cpp index 42e1117..1ebc141 100644 --- a/core/sql/common/ComObjectName.cpp +++ b/core/sql/common/ComObjectName.cpp @@ -434,9 +434,12 @@ ComObjectName::operator= (const ComObjectName &rhsObjectName) // const NAString & -ComObjectName::getCatalogNamePartAsAnsiString(NABoolean) const +ComObjectName::getCatalogNamePartAsAnsiString(NABoolean v) const { - return catalogNamePart_.getExternalName(); + if (v == FALSE) + return catalogNamePart_.getExternalName(); + else + return catalogNamePart_.getInternalName(); } const NAString & http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/common/NAString.cpp ---------------------------------------------------------------------- diff --git a/core/sql/common/NAString.cpp b/core/sql/common/NAString.cpp index b0bfb0b..5733837 100644 --- a/core/sql/common/NAString.cpp +++ b/core/sql/common/NAString.cpp @@ -560,7 +560,12 @@ Lng32 ToInternalIdentifier( NAString &ansiIdent size_t countOfRemoved = i; i = 0; - if (ansiIdent[i] != '"') { // REGULAR identifier + // Handle double quotes or backquotes as delimited identifiers. + // Backquotes are used for hive objects. + // An error will be returned later if they are used for traf objects. + NABoolean isDquote = (ansiIdent[i] == '"'); + if ((ansiIdent[i] != '"') && + (ansiIdent[i] != '`')) { // REGULAR identifier // ANSI 5.2 SR 13 + 14 and 8.2 SR 3a say that trailing spaces are // insignificant in equality-testing of identifiers, so remove them @@ -759,7 +764,7 @@ Lng32 ToInternalIdentifier( NAString &ansiIdent } // end REGULAR identifier else { - UInt32 state = 1; + UInt32 state = (isDquote ? 1 : 3); ansiIdent.remove(0,1); // remove initial dquote countOfRemoved++; @@ -937,12 +942,28 @@ Lng32 ToInternalIdentifier( NAString &ansiIdent return illegalCharInIdentifier(ansiIdent, i, countOfRemoved); i++; break; + case 3: + if (c == '`') { + ansiIdent.remove(i,1); + countOfRemoved++; + state = 4; + } else + i++; + break; + case 4: + if (c == '`') + state = 3; + else if (c != ' ') // tab became space + return illegalCharInIdentifier(ansiIdent, i, countOfRemoved); + i++; + break; default: ComASSERT(FALSE); } // switch } // while - if (state != 2) + if ((isDquote && (state != 2)) || + (NOT isDquote && (state != 4))) return illegalCharInIdentifier(ansiIdent, i, countOfRemoved); // ANSI 5.2 SR 13 + 14 and 8.2 SR 3a say that trailing spaces http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/common/NAType.cpp ---------------------------------------------------------------------- diff --git a/core/sql/common/NAType.cpp b/core/sql/common/NAType.cpp index 0271b0c..b428500 100644 --- a/core/sql/common/NAType.cpp +++ b/core/sql/common/NAType.cpp @@ -690,7 +690,15 @@ short NAType::getMyTypeAsHiveText(NAString * outputStr/*out*/) const switch (fs_datatype) { case REC_MIN_F_CHAR_H ... REC_MAX_F_CHAR_H: - *outputStr = "string"; + { + SQLChar * ct = (SQLChar*)this; + char buf[20]; + Int32 size = ct->getStrCharLimit(); + str_itoa(size, buf); + *outputStr = "char("; + *outputStr += buf; + *outputStr += ")"; + } break; case REC_MIN_V_CHAR_H ... REC_MAX_V_CHAR_H: http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/common/OperTypeEnum.h ---------------------------------------------------------------------- diff --git a/core/sql/common/OperTypeEnum.h b/core/sql/common/OperTypeEnum.h index 404c704..135264c 100644 --- a/core/sql/common/OperTypeEnum.h +++ b/core/sql/common/OperTypeEnum.h @@ -971,6 +971,9 @@ enum OperatorTypeEnum { DDL_INITIALIZE_SECURITY, DDL_COMMENT_ON, + // ddl operations on hive objects + DDL_ON_HIVE_OBJECTS, + // // Elements in DDL statements // @@ -1093,6 +1096,7 @@ enum OperatorTypeEnum { ELM_LIKE_OPT_LIMIT_COLUMN_LENGTH, ELM_LIKE_OPT_WITHOUT_ROW_FORMAT_ELEM, ELM_LIKE_OPT_WITHOUT_LOB_COLUMNS, + ELM_LIKE_OPT_WITH_HIVE_OPTIONS, ELM_LOCATION_ELEM, ELM_OPTION_LIST, ELM_PARALLEL_EXEC_ELEM, http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/executor/ExComTdb.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExComTdb.cpp b/core/sql/executor/ExComTdb.cpp index 3766b3d..68b4cf7 100644 --- a/core/sql/executor/ExComTdb.cpp +++ b/core/sql/executor/ExComTdb.cpp @@ -347,7 +347,6 @@ char *ComTdb::findVTblPtrExe(short classID) GetVTblPtr(vtblptr,ExExeUtilHiveQueryTdb); break; } - case ex_PROCESS_VOLATILE_TABLE: { GetVTblPtr(vtblptr,ExProcessVolatileTableTdb); http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/executor/ExExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h index a3f197d..84868ab 100644 --- a/core/sql/executor/ExExeUtil.h +++ b/core/sql/executor/ExExeUtil.h @@ -3516,75 +3516,26 @@ class ExExeUtilHiveTruncatePrivateState : public ex_tcb_private_state protected: }; -// ----------------------------------------------------------------------- -// ExExeUtilHiveQueryTdb -// ----------------------------------------------------------------------- class ExExeUtilHiveQueryTdb : public ComTdbExeUtilHiveQuery { public: - - // --------------------------------------------------------------------- - // Constructor is only called to instantiate an object used for - // retrieval of the virtual table function pointer of the class while - // unpacking. An empty constructor is enough. - // --------------------------------------------------------------------- ExExeUtilHiveQueryTdb() {} - virtual ~ExExeUtilHiveQueryTdb() {} - - // --------------------------------------------------------------------- - // Build a TCB for this TDB. Redefined in the Executor project. - // --------------------------------------------------------------------- virtual ex_tcb *build(ex_globals *globals); - private: - // --------------------------------------------------------------------- - // !!!!!!! IMPORTANT -- NO DATA MEMBERS ALLOWED IN EXECUTOR TDB !!!!!!!! - // ********************************************************************* - // The Executor TDB's are only used for the sole purpose of providing a - // way to supplement the Compiler TDB's (in comexe) with methods whose - // implementation depends on Executor objects. This is done so as to - // decouple the Compiler from linking in Executor objects unnecessarily. - // - // When a Compiler generated TDB arrives at the Executor, the same data - // image is "cast" as an Executor TDB after unpacking. Therefore, it is - // a requirement that a Compiler TDB has the same object layout as its - // corresponding Executor TDB. As a result of this, all Executor TDB's - // must have absolutely NO data members, but only member functions. So, - // if you reach here with an intention to add data members to a TDB, ask - // yourself two questions: - // - // 1. Are those data members Compiler-generated? - // If yes, put them in the ComTdbDLL instead. - // If no, they should probably belong to someplace else (like TCB). - // - // 2. Are the classes those data members belong defined in the executor - // project? - // If your answer to both questions is yes, you might need to move - // the classes to the comexe project. - // --------------------------------------------------------------------- }; - -/////////////////////////////////////////////////////////////// -// ExExeUtilHiveQueryTcb -/////////////////////////////////////////////////////////////// class ExExeUtilHiveQueryTcb : public ExExeUtilTcb { public: - // Constructor ExExeUtilHiveQueryTcb(const ComTdbExeUtilHiveQuery & exe_util_tdb, ex_globals * glob = 0); - ~ExExeUtilHiveQueryTcb(); - virtual short work(); - virtual ex_tcb_private_state * allocatePstates( Lng32 &numElems, // inout, desired/actual elements Lng32 &pstateLength); // out, length of one element - private: enum Step { @@ -3593,23 +3544,18 @@ class ExExeUtilHiveQueryTcb : public ExExeUtilTcb PROCESS_QUERY_, DONE_ }; - ExExeUtilHiveQueryTdb & htTdb() const {return (ExExeUtilHiveQueryTdb &) tdb;}; - Step step_; }; - class ExExeUtilHiveQueryPrivateState : public ex_tcb_private_state { friend class ExExeUtilHiveQueryTcb; - public: ExExeUtilHiveQueryPrivateState(); ~ExExeUtilHiveQueryPrivateState(); // destructor protected: }; - ////////////////////////////////////////////////////////////////////////// // ----------------------------------------------------------------------- // ExExeUtilHbaseLoadTdb http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/executor/ExExeUtilMisc.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtilMisc.cpp b/core/sql/executor/ExExeUtilMisc.cpp index 6568f12..dd4cd35 100644 --- a/core/sql/executor/ExExeUtilMisc.cpp +++ b/core/sql/executor/ExExeUtilMisc.cpp @@ -2454,58 +2454,34 @@ ExExeUtilHiveTruncatePrivateState::~ExExeUtilHiveTruncatePrivateState() { }; -//////////////////////////////////////////////////////////////// -// Constructor for class ExExeUtilHiveQueryTdb -/////////////////////////////////////////////////////////////// ex_tcb * ExExeUtilHiveQueryTdb::build(ex_globals * glob) { ExExeUtilTcb * exe_util_tcb; - exe_util_tcb = new(glob->getSpace()) ExExeUtilHiveQueryTcb(*this, glob); exe_util_tcb->registerSubtasks(); - return (exe_util_tcb); } - - -//////////////////////////////////////////////////////////////// -// Constructor for class ExExeUtilHiveQueryTcb -/////////////////////////////////////////////////////////////// ExExeUtilHiveQueryTcb::ExExeUtilHiveQueryTcb( const ComTdbExeUtilHiveQuery & exe_util_tdb, ex_globals * glob) : ExExeUtilTcb( exe_util_tdb, NULL, glob) { - // Allocate the private state in each entry of the down queue qparent_.down->allocatePstate(this); - step_ = INITIAL_; } - ExExeUtilHiveQueryTcb::~ExExeUtilHiveQueryTcb() { } - -////////////////////////////////////////////////////// -// work() for ExExeUtilHiveQueryTsb -////////////////////////////////////////////////////// short ExExeUtilHiveQueryTcb::work() { short rc = 0; Lng32 cliRC = 0; - - // if no parent request, return if (qparent_.down->isEmpty()) return WORK_OK; - - // if no room in up queue, won't be able to return data/status. - // Come back later. if (qparent_.up->isFull()) return WORK_OK; - ex_queue_entry * pentry_down = qparent_.down->getHeadEntry(); ExExeUtilPrivateState & pstate = *((ExExeUtilPrivateState*) pentry_down->pstate); - while (1) { switch (step_) @@ -2515,7 +2491,6 @@ short ExExeUtilHiveQueryTcb::work() step_ = PROCESS_QUERY_; } break; - case PROCESS_QUERY_: { if (HiveClient_JNI::executeHiveSQL(htTdb().getHiveQuery()) != HVC_OK) @@ -2529,7 +2504,6 @@ short ExExeUtilHiveQueryTcb::work() step_ = DONE_; } break; - case ERROR_: { if (handleError()) @@ -2537,40 +2511,27 @@ short ExExeUtilHiveQueryTcb::work() step_ = DONE_; } break; - case DONE_: { if (handleDone()) return WORK_OK; - step_ = INITIAL_; - return WORK_OK; } break; - } // switch } // while - } - - ex_tcb_private_state * ExExeUtilHiveQueryTcb::allocatePstates( Lng32 &numElems, // inout, desired/actual elements Lng32 &pstateLength) // out, length of one element { PstateAllocator<ExExeUtilHiveQueryPrivateState> pa; - return pa.allocatePstates(this, numElems, pstateLength); } - -///////////////////////////////////////////////////////////////////////////// -// Constructor and destructor for ExeUtil_private_state -///////////////////////////////////////////////////////////////////////////// ExExeUtilHiveQueryPrivateState::ExExeUtilHiveQueryPrivateState() { } - ExExeUtilHiveQueryPrivateState::~ExExeUtilHiveQueryPrivateState() { }; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/executor/HiveClient_JNI.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/HiveClient_JNI.cpp b/core/sql/executor/HiveClient_JNI.cpp index 9c09b97..80248a8 100644 --- a/core/sql/executor/HiveClient_JNI.cpp +++ b/core/sql/executor/HiveClient_JNI.cpp @@ -436,7 +436,8 @@ HVC_RetCode HiveClient_JNI::executeHiveSQL(const char* hiveSQL) if (jenv_->ExceptionCheck()) { - getExceptionDetails(__FILE__, __LINE__, "HiveClient_JNI::executeHiveSQL()"); + getExceptionDetails(__FILE__, __LINE__, "HiveClient_JNI::executeHiveSQL()", + TRUE /*dont return stack details*/); jenv_->PopLocalFrame(NULL); return HVC_ERROR_EXECUTE_HIVE_SQL_EXCEPTION; } http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/executor/JavaObjectInterface.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/JavaObjectInterface.cpp b/core/sql/executor/JavaObjectInterface.cpp index 282e40b..700c3c1 100644 --- a/core/sql/executor/JavaObjectInterface.cpp +++ b/core/sql/executor/JavaObjectInterface.cpp @@ -566,7 +566,8 @@ void JavaObjectInterface::logError(std::string &cat, const char* file, int line) } NABoolean JavaObjectInterface::getExceptionDetails(const char *fileName, int lineNo, - const char *methodName) + const char *methodName, + NABoolean noDetails) { JNIEnv *jenv = jenv_; CliGlobals *cliGlobals = GetCliGlobals(); @@ -585,7 +586,7 @@ NABoolean JavaObjectInterface::getExceptionDetails(const char *fileName, int li setSqlJniErrorStr(error_msg); return FALSE; } - appendExceptionMessages(a_exception, error_msg); + appendExceptionMessages(a_exception, error_msg, noDetails); setSqlJniErrorStr(error_msg); logError(CAT_SQL_EXE, fileName, lineNo); logError(CAT_SQL_EXE, methodName, error_msg); @@ -593,7 +594,9 @@ NABoolean JavaObjectInterface::getExceptionDetails(const char *fileName, int li return TRUE; } -void JavaObjectInterface::appendExceptionMessages(jthrowable a_exception, NAString &error_msg) +void JavaObjectInterface::appendExceptionMessages(jthrowable a_exception, + NAString &error_msg, + NABoolean noDetails) { jstring msg_obj = (jstring) jenv_->CallObjectMethod(a_exception, @@ -617,6 +620,10 @@ void JavaObjectInterface::appendExceptionMessages(jthrowable a_exception, NAStri gGetStackTraceMethodID); if (frames == NULL) return; + + if (noDetails) + return; + jsize frames_length = jenv_->GetArrayLength(frames); jsize i = 0; @@ -638,7 +645,7 @@ void JavaObjectInterface::appendExceptionMessages(jthrowable a_exception, NAStri jthrowable j_cause = (jthrowable)jenv_->CallObjectMethod(a_exception, gGetCauseMethodID); if (j_cause != NULL) { error_msg += " Caused by \n"; - appendExceptionMessages(j_cause, error_msg); + appendExceptionMessages(j_cause, error_msg, noDetails); } jenv_->DeleteLocalRef(a_exception); } http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/executor/JavaObjectInterface.h ---------------------------------------------------------------------- diff --git a/core/sql/executor/JavaObjectInterface.h b/core/sql/executor/JavaObjectInterface.h index f6da8ec..9c04082 100644 --- a/core/sql/executor/JavaObjectInterface.h +++ b/core/sql/executor/JavaObjectInterface.h @@ -148,9 +148,12 @@ public: return isInitialized_; } static NABoolean getExceptionDetails(const char *fileName, int lineNo, - const char *methodName); + const char *methodName, + NABoolean noDetails = FALSE); - static void appendExceptionMessages(jthrowable a_exception, NAString &error_msg); + static void appendExceptionMessages(jthrowable a_exception, + NAString &error_msg, + NABoolean noDetails = FALSE); NAHeap *getHeap() { return heap_; } protected: http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/export/ComDiags.cpp ---------------------------------------------------------------------- diff --git a/core/sql/export/ComDiags.cpp b/core/sql/export/ComDiags.cpp index b3c6169..a8d8e79 100644 --- a/core/sql/export/ComDiags.cpp +++ b/core/sql/export/ComDiags.cpp @@ -2753,6 +2753,7 @@ void ComDiagsArea::insertNewEODWarning() void ComDiagsArea::insertNewError() { + errors_.insert(newCondition_); // for non-atomic inserts, if any error is inserted after NonFatalErrorSeen flag // is set, then we want to unset the flag so that mainSQLCODE does not return 30022. http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/generator/GenExplain.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenExplain.cpp b/core/sql/generator/GenExplain.cpp index 81d1e75..6234952 100644 --- a/core/sql/generator/GenExplain.cpp +++ b/core/sql/generator/GenExplain.cpp @@ -62,6 +62,7 @@ #include "StmtDDLCreateTable.h" #include "StmtDDLCreateIndex.h" +#include "StmtDDLonHiveObjects.h" #include "ComDistribution.h" #include "TrafDDLdesc.h" @@ -1123,7 +1124,29 @@ DDLExpr::addSpecificExplainInfo(ExplainTupleMaster *explainTuple, char buf[200]; NAString buffer; - buffer = "explain_information: not available."; + ExprNode *ddlNode = getDDLNode(); + if (ddlNode) + { + if (ddlNode->getOperatorType() == DDL_ON_HIVE_OBJECTS) + { + StmtDDLonHiveObjects * hddl = + ddlNode->castToStmtDDLNode()->castToStmtDDLonHiveObjects(); + buffer = "explain_information: DDL on Hive object "; + buffer += NAString("ddl_operation: ") + hddl->getOperStr() + " "; + if (NOT hddl->getName().isNull()) + buffer += NAString("object_name: ") + hddl->getName() + " "; + else + buffer += "object_name: unknown "; + buffer += NAString("object_type: ") + hddl->getTypeStr() + " "; + if (NOT hddl->getHiveDDL().isNull()) + buffer += NAString("hive_ddl: ") + hddl->getHiveDDL() + " "; + else + buffer += "hive_ddl: unknown "; + } + } // ddlNode + + if (buffer.isNull()) + buffer = "explain_information: not available."; explainTuple->setDescription(buffer); http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/generator/GenFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenFastTransport.cpp b/core/sql/generator/GenFastTransport.cpp index 5019485..7080f34 100644 --- a/core/sql/generator/GenFastTransport.cpp +++ b/core/sql/generator/GenFastTransport.cpp @@ -665,6 +665,12 @@ PhysicalFastExtract::codeGen(Generator *generator) } } // do sim check + if (getHiveTableDesc() && + getHiveTableDesc()->getNATable() && + getHiveTableDesc()->getNATable()->isEnabledForDDLQI()) + generator->objectUids().insert( + getHiveTableDesc()->getNATable()->objectUid().get_value()); + targetName = AllocStringInSpace(*space, (char *)getTargetName().data()); hdfsHostName = AllocStringInSpace(*space, (char *)getHdfsHostName().data()); hiveTableName = AllocStringInSpace(*space, (char *)getHiveTableName().data()); http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/generator/GenPreCode.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenPreCode.cpp b/core/sql/generator/GenPreCode.cpp index 9832c44..10db874 100644 --- a/core/sql/generator/GenPreCode.cpp +++ b/core/sql/generator/GenPreCode.cpp @@ -2860,7 +2860,8 @@ short DDLExpr::ddlXnsInfo(NABoolean &isDDLxn, NABoolean &xnCanBeStarted) (ddlNode->getOperatorType() == DDL_ALTER_TABLE_ALTER_COLUMN_DATATYPE) || (ddlNode->getOperatorType() == DDL_ALTER_TABLE_ALTER_HBASE_OPTIONS) || (ddlNode->getOperatorType() == DDL_ALTER_INDEX_ALTER_HBASE_OPTIONS) || - (ddlNode->getOperatorType() == DDL_ALTER_TABLE_RENAME))) + (ddlNode->getOperatorType() == DDL_ALTER_TABLE_RENAME) || + (ddlNode->getOperatorType() == DDL_ON_HIVE_OBJECTS))) { // transaction will be started and commited in called methods. xnCanBeStarted = FALSE; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/generator/GenRelExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelExeUtil.cpp b/core/sql/generator/GenRelExeUtil.cpp index 1b0adb4..2c043d2 100644 --- a/core/sql/generator/GenRelExeUtil.cpp +++ b/core/sql/generator/GenRelExeUtil.cpp @@ -3404,40 +3404,27 @@ short ExeUtilHiveTruncate::codeGen(Generator * generator) return 0; } -///////////////////////////////////////////////////////// -// -// ExeUtilHiveQuery::codeGen() -// -///////////////////////////////////////////////////////// short ExeUtilHiveQuery::codeGen(Generator * generator) { ExpGenerator * expGen = generator->getExpGenerator(); Space * space = generator->getSpace(); - - // allocate a map table for the retrieved columns generator->appendAtEnd(); - ex_cri_desc * givenDesc = generator->getCriDesc(Generator::DOWN); - ex_cri_desc * returnedDesc = new(space) ex_cri_desc(givenDesc->noTuples() + 1, space); - ex_cri_desc * workCriDesc = new(space) ex_cri_desc(4, space); const Int32 work_atp = 1; const Int32 exe_util_row_atp_index = 2; - short rc = processOutputRow(generator, work_atp, exe_util_row_atp_index, returnedDesc); if (rc) { return -1; } - char * hive_query = space->AllocateAndCopyToAlignedSpace (hiveQuery(), 0); Lng32 hive_query_len = hiveQuery().length(); - ComTdbExeUtilHiveQuery * exe_util_tdb = new(space) ComTdbExeUtilHiveQuery(hive_query, hive_query_len, @@ -3447,24 +3434,17 @@ short ExeUtilHiveQuery::codeGen(Generator * generator) (queue_index)getDefault(GEN_DDL_SIZE_UP), getDefault(GEN_DDL_NUM_BUFFERS), getDefault(GEN_DDL_BUFFER_SIZE)); - generator->initTdbFields(exe_util_tdb); - if(!generator->explainDisabled()) { generator->setExplainTuple( addExplainInfo(exe_util_tdb, 0, 0, generator)); } - - // no tupps are returned generator->setCriDesc((ex_cri_desc *)(generator->getCriDesc(Generator::DOWN)), Generator::UP); generator->setGenObj(this, exe_util_tdb); - generator->setTransactionFlag(0); // transaction is not needed. - return 0; } - //////////////////////////////////////////////////////////////////// // class ExeUtilRegionStats //////////////////////////////////////////////////////////////////// http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/generator/GenRelScan.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelScan.cpp b/core/sql/generator/GenRelScan.cpp index 0a133b2..608ecb0 100644 --- a/core/sql/generator/GenRelScan.cpp +++ b/core/sql/generator/GenRelScan.cpp @@ -1319,6 +1319,10 @@ if (hTabStats->isOrcFile()) } } + if (getTableDesc()->getNATable()->isEnabledForDDLQI()) + generator->objectUids().insert( + getTableDesc()->getNATable()->objectUid().get_value()); + // create hdfsscan_tdb ComTdbHdfsScan *hdfsscan_tdb = new(space) ComTdbHdfsScan( http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/optimizer/BindRelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp index d5e874b..58d2a4e 100644 --- a/core/sql/optimizer/BindRelExpr.cpp +++ b/core/sql/optimizer/BindRelExpr.cpp @@ -1653,7 +1653,7 @@ NATable *BindWA::getNATable(CorrName& corrName, // allowExternalTables is set for drop table and SHOWDDL statements. // TDB - may want to merge the Trafodion version with the native version. if ((table) && - (table->isExternalTable() && + (table->isTrafExternalTable() && (NOT table->getTableName().isHbaseMappedName()) && (! bindWA->allowExternalTables()))) { @@ -1667,7 +1667,7 @@ NATable *BindWA::getNATable(CorrName& corrName, // If the table is an external table and has an associated native table, // check to see if the external table structure still matches the native table. // If not, return an error - if ((table) && table->isExternalTable() && + if ((table) && table->isTrafExternalTable() && (NOT table->getTableName().isHbaseMappedName())) { NAString adjustedName =ComConvertTrafNameToNativeName @@ -2119,6 +2119,7 @@ RelExpr *BindWA::bindView(const CorrName &viewName, } Parser parser(bindWA->currentCmpContext()); + parser.hiveDDLInfo_->disableDDLcheck_ = TRUE; ExprNode *viewTree = parser.parseDML(naTable->getViewText(), naTable->getViewLen(), naTable->getViewTextCharSet()); http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/optimizer/NATable.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp index 0ad9b3f..d330c3b 100644 --- a/core/sql/optimizer/NATable.cpp +++ b/core/sql/optimizer/NATable.cpp @@ -5093,7 +5093,7 @@ NABoolean NATable::fetchObjectUIDForNativeTable(const CorrName& corrName, if (corrName.isExternal()) { - setIsExternalTable(TRUE); + setIsTrafExternalTable(TRUE); } if (qualifiedName_.getQualifiedNameObj().isHistograms() || @@ -5146,10 +5146,10 @@ NABoolean NATable::fetchObjectUIDForNativeTable(const CorrName& corrName, if ((table_desc->tableDesc()->objectFlags & SEABASE_OBJECT_IS_EXTERNAL_HIVE) != 0 || (table_desc->tableDesc()->objectFlags & SEABASE_OBJECT_IS_EXTERNAL_HBASE) != 0) { - setIsExternalTable(TRUE); + setIsTrafExternalTable(TRUE); if (table_desc->tableDesc()->objectFlags & SEABASE_OBJECT_IS_IMPLICIT_EXTERNAL) - setIsImplicitExternalTable(TRUE); + setIsImplicitTrafExternalTable(TRUE); } if (CmpSeabaseDDL::isMDflagsSet @@ -5840,6 +5840,12 @@ NATable::NATable(BindWA *bindWA, viewExpandedText = replaceAll(viewExpandedText, "`", ""); NAString createViewStmt("CREATE VIEW "); + createViewStmt += NAString("hive."); + if (strcmp(htbl->schName_, "default") == 0) + createViewStmt += "hive"; + else + createViewStmt += htbl->schName_; + createViewStmt += "."; createViewStmt += htbl->tblName_ + NAString(" AS ") + viewExpandedText + NAString(";"); @@ -5861,6 +5867,11 @@ NATable::NATable(BindWA *bindWA, } else { + if (htbl->isExternalTable()) + setIsHiveExternalTable(TRUE); + else if (htbl->isManagedTable()) + setIsHiveManagedTable(TRUE); + if (createNAFileSets(htbl /*IN*/, this /*IN*/, colArray_ /*IN*/, @@ -6921,7 +6932,9 @@ bool NATable::isEnabledForDDLQI() const { if (isSeabaseMD_ || isSMDTable_ || (getSpecialType() == ExtendedQualName::VIRTUAL_TABLE)) return false; - else + else if (isHiveTable() && (objectUID_.get_value() == 0)) + return false; + else { if (objectUID_.get_value() == 0) { @@ -8322,7 +8335,7 @@ NATable * NATableDB::get(CorrName& corrName, BindWA * bindWA, if (isHbaseMap) { table->setIsHbaseMapTable(TRUE); - table->setIsExternalTable(TRUE); + table->setIsTrafExternalTable(TRUE); } } else if (isHiveTable(corrName) && @@ -8491,7 +8504,8 @@ NATable * NATableDB::get(CorrName& corrName, BindWA * bindWA, { *CmpCommon::diags() << DgSqlCode(-1388) - << DgTableName(corrName.getExposedNameAsAnsiString()); + << DgString0("Object") + << DgString1(corrName.getExposedNameAsAnsiString()); } } else http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/optimizer/NATable.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/NATable.h b/core/sql/optimizer/NATable.h index 4feece9..d793335 100644 --- a/core/sql/optimizer/NATable.h +++ b/core/sql/optimizer/NATable.h @@ -718,17 +718,17 @@ public: NABoolean hasSerializedColumn() const { return (flags_ & SERIALIZED_COLUMN) != 0; } - void setIsExternalTable( NABoolean value ) - { value ? flags_ |= IS_EXTERNAL_TABLE : flags_ &= ~IS_EXTERNAL_TABLE; } + void setIsTrafExternalTable( NABoolean value ) + { value ? flags_ |= IS_TRAF_EXTERNAL_TABLE : flags_ &= ~IS_TRAF_EXTERNAL_TABLE; } - NABoolean isExternalTable() const - { return (flags_ & IS_EXTERNAL_TABLE) != 0; } + NABoolean isTrafExternalTable() const + { return (flags_ & IS_TRAF_EXTERNAL_TABLE) != 0; } - void setIsImplicitExternalTable( NABoolean value ) - { value ? flags_ |= IS_IMPLICIT_EXTERNAL_TABLE : flags_ &= ~IS_IMPLICIT_EXTERNAL_TABLE; } + void setIsImplicitTrafExternalTable( NABoolean value ) + { value ? flags_ |= IS_IMPLICIT_TRAF_EXT_TABLE : flags_ &= ~IS_IMPLICIT_TRAF_EXT_TABLE; } - NABoolean isImplicitExternalTable() const - { return (flags_ & IS_IMPLICIT_EXTERNAL_TABLE) != 0; } + NABoolean isImplicitTrafExternalTable() const + { return (flags_ & IS_IMPLICIT_TRAF_EXT_TABLE) != 0; } void setHasExternalTable( NABoolean value ) { value ? flags_ |= HAS_EXTERNAL_TABLE : flags_ &= ~HAS_EXTERNAL_TABLE; } @@ -780,6 +780,16 @@ public: NABoolean isInternalRegistered() const { return (flags_ & IS_INTERNAL_REGISTERED) != 0; } + + void setIsHiveExternalTable( NABoolean value ) + { value ? flags_ |= IS_HIVE_EXTERNAL_TABLE : flags_ &= ~IS_HIVE_EXTERNAL_TABLE; } + NABoolean isHiveExternalTable() const + { return (flags_ & IS_HIVE_EXTERNAL_TABLE) != 0; } + + void setIsHiveManagedTable( NABoolean value ) + { value ? flags_ |= IS_HIVE_MANAGED_TABLE : flags_ &= ~IS_HIVE_MANAGED_TABLE; } + NABoolean isHiveManagedTable() const + { return (flags_ & IS_HIVE_MANAGED_TABLE) != 0; } const CheckConstraintList &getCheckConstraints() const { return checkConstraints_; } @@ -997,7 +1007,7 @@ private: REMOVE_FROM_CACHE_BNC = 0x00010000, // Remove from NATable Cache Before Next Compilation SERIALIZED_ENCODED_COLUMN = 0x00020000, SERIALIZED_COLUMN = 0x00040000, - IS_EXTERNAL_TABLE = 0x00080000, + IS_TRAF_EXTERNAL_TABLE = 0x00080000, HAS_EXTERNAL_TABLE = 0x00100000, IS_HISTOGRAM_TABLE = 0x00200000, HBASE_MAP_TABLE = 0x00400000, @@ -1005,9 +1015,18 @@ private: HAS_HIVE_EXT_TABLE = 0x01000000, HIVE_EXT_COL_ATTRS = 0x02000000, HIVE_EXT_KEY_ATTRS = 0x04000000, - IS_IMPLICIT_EXTERNAL_TABLE= 0x08000000, + IS_IMPLICIT_TRAF_EXT_TABLE= 0x08000000, IS_REGISTERED = 0x10000000, - IS_INTERNAL_REGISTERED = 0x20000000 + IS_INTERNAL_REGISTERED = 0x20000000, + + // if underlying hive table was created as an EXTERNAL table. + // hive syntax: create external table ...) + // Note: this is different than a traf external table created for + // a hive table. + IS_HIVE_EXTERNAL_TABLE = 0x40000000, + + // if underlying hive table was not created as an EXTERNAL table. + IS_HIVE_MANAGED_TABLE = 0x80000000, }; UInt32 flags_; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/optimizer/RelExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExeUtil.cpp b/core/sql/optimizer/RelExeUtil.cpp index 5d143a2..1158d4e 100644 --- a/core/sql/optimizer/RelExeUtil.cpp +++ b/core/sql/optimizer/RelExeUtil.cpp @@ -84,6 +84,7 @@ #include "StmtDDLAlterLibrary.h" #include "StmtDDLRegOrUnregHive.h" #include "StmtDDLCommentOn.h" +#include "StmtDDLonHiveObjects.h" #include <cextdecs/cextdecs.h> #include "wstr.h" @@ -275,6 +276,9 @@ const NAString DDLExpr::getText() const { NAString result(CmpCommon::statementHeap()); + if (getDDLNode() && getDDLNode()->getOperatorType() == DDL_ON_HIVE_OBJECTS) + result = "HIVE_DDL"; + else result = "DDL"; return result; @@ -447,8 +451,11 @@ const NAString ExeUtilExpr::getText() const result = "REGION_STATS"; break; - default: - + case HIVE_QUERY_: + result = "HIVE_QUERY"; + break; + + default: result = "ADD_TO_EXEUTILEXPR::GETTEXT()"; break; @@ -896,13 +903,10 @@ RelExpr * ExeUtilHiveTruncate::copyTopNode(RelExpr *derivedNode, CollHeap* outHe // ----------------------------------------------------------------------- -// Member functions for class ExeUtilHiveQuery -// ----------------------------------------------------------------------- RelExpr * ExeUtilHiveQuery::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap) { ExeUtilHiveQuery *result; - if (derivedNode == NULL) result = new (outHeap) ExeUtilHiveQuery(hiveQuery(), @@ -910,36 +914,24 @@ RelExpr * ExeUtilHiveQuery::copyTopNode(RelExpr *derivedNode, outHeap); else result = (ExeUtilHiveQuery *) derivedNode; - return ExeUtilExpr::copyTopNode(result, outHeap); } - RelExpr * ExeUtilHiveQuery::bindNode(BindWA *bindWA) { if (type_ != FROM_STRING) { - // error case *CmpCommon::diags() << DgSqlCode(-3242) << DgString0("DDL can only be specified as a string."); - bindWA->setErrStatus(); return NULL; } - - // currently supported hive queries must start with: - // create, drop, alter, truncate - // Check for it. - - // first strip leading spaces. hiveQuery_ = hiveQuery_.strip(NAString::leading, ' '); - if (NOT ((hiveQuery_.index("CREATE", 0, NAString::ignoreCase) == 0) || - (hiveQuery_.index("DROP", 0, NAString::ignoreCase) == 0) || - (hiveQuery_.index("ALTER", 0, NAString::ignoreCase) == 0) || - // (hiveQuery_.index("INSERT", 0, NAString::ignoreCase) == 0) || - (hiveQuery_.index("TRUNCATE", 0, NAString::ignoreCase) == 0))) - { - // error case - *CmpCommon::diags() << DgSqlCode(-3242) << DgString0("Only CREATE, DROP, ALTER or TRUNCATE hive DDL statements can be specified."); - + if (NOT ((hiveQuery_.index("CREATE ", 0, NAString::ignoreCase) == 0) || + (hiveQuery_.index("DROP ", 0, NAString::ignoreCase) == 0) || + (hiveQuery_.index("ALTER ", 0, NAString::ignoreCase) == 0) || + (hiveQuery_.index("INSERT ", 0, NAString::ignoreCase) == 0) || + (hiveQuery_.index("TRUNCATE ", 0, NAString::ignoreCase) == 0))) + { + *CmpCommon::diags() << DgSqlCode(-3242) << DgString0("Specified operation cannot be executed directly by Hive."); bindWA->setErrStatus(); return NULL; } @@ -947,11 +939,8 @@ RelExpr * ExeUtilHiveQuery::bindNode(BindWA *bindWA) RelExpr * boundExpr = ExeUtilExpr::bindNode(bindWA); if (bindWA->errStatus()) return NULL; - return boundExpr; } - -// ----------------------------------------------------------------------- // Member functions for class ExeUtilGetStatistics // ----------------------------------------------------------------------- RelExpr * ExeUtilGetStatistics::copyTopNode(RelExpr *derivedNode, @@ -4085,6 +4074,7 @@ RelExpr * DDLExpr::bindNode(BindWA *bindWA) NABoolean isVolatile = FALSE; NABoolean isRegister = FALSE; NABoolean isCommentOn = FALSE; + NABoolean isHive = FALSE; returnStatus_ = FALSE; @@ -4170,7 +4160,7 @@ RelExpr * DDLExpr::bindNode(BindWA *bindWA) externalTable = TRUE; isHbase_ = TRUE; } - else + else { *CmpCommon::diags() << DgSqlCode(-3242) << DgString0("External tables supported on hive tables only."); bindWA->setErrStatus(); @@ -4559,6 +4549,21 @@ RelExpr * DDLExpr::bindNode(BindWA *bindWA) qualObjName_ = getExprNode()->castToStmtDDLNode()-> castToStmtDDLCommentOn()->getObjectNameAsQualifiedName(); } + else if (getExprNode()->castToStmtDDLNode()->castToStmtDDLonHiveObjects()) + { + if (getExprNode()->castToStmtDDLNode()->castToStmtDDLonHiveObjects()->getOper() != StmtDDLonHiveObjects::PASSTHRU_DDL_) + { + if (CmpCommon::getDefault(TRAF_DDL_ON_HIVE_OBJECTS) == DF_OFF) + { + *CmpCommon::diags() << DgSqlCode(-3242) << DgString0("DDL on Hive objects is not allowed from Trafodion interface. Use hive shell to perform this operation."); + bindWA->setErrStatus(); + return NULL; + } + } + + isHive = TRUE; + isHbase_ = TRUE; + } if (isCleanup_) { @@ -4567,7 +4572,7 @@ RelExpr * DDLExpr::bindNode(BindWA *bindWA) } if ((isCreateSchema || isDropSchema || isAlterSchema) || isRegister || isCommentOn || - ((isTable_ || isIndex_ || isView_ || isRoutine_ || isLibrary_ || isSeq) && + ((isTable_ || isIndex_ || isView_ || isRoutine_ || isLibrary_ || isSeq || isHive) && (isCreate_ || isDrop_ || purgedata() || (isAlter_ && (alterAddCol || alterDropCol || alterDisableIndex || alterEnableIndex || alterAddConstr || alterDropConstr || alterRenameTable || @@ -4616,7 +4621,7 @@ RelExpr * DDLExpr::bindNode(BindWA *bindWA) isHbase_ = TRUE; } - else + else if (NOT isHive) { if ((alterDropCol) || (alterEnableIndex)) { @@ -4637,13 +4642,27 @@ RelExpr * DDLExpr::bindNode(BindWA *bindWA) if (bindWA->errStatus()) return NULL; - if (isHbase_ || externalTable || isVolatile) + if (isHbase_ || externalTable || isVolatile || isHive) return boundExpr; if (isView_ && (isCreate_ || isDrop_)) - *CmpCommon::diags() << DgSqlCode(-3242) << DgString0("DDL views can only be created or dropped in trafodion schema."); + { + if (qualObjName_.getCatalogName().isNull()) + *CmpCommon::diags() << DgSqlCode(-3242) << DgString0("This view cannot be created or dropped."); + else + *CmpCommon::diags() << DgSqlCode(-3242) << DgString0(NAString("This view cannot be created or dropped in the specified catalog '") + qualObjName_.getCatalogName() + "'."); + } + else if ((NOT qualObjName_.getCatalogName().isNull()) && + (NOT ((qualObjName_.getCatalogName() == TRAFODION_SYSCAT_LIT) || + (qualObjName_.getCatalogName() == HBASE_SYSTEM_SCHEMA) || + (qualObjName_.getCatalogName() == HIVE_SYSTEM_SCHEMA)))) + { + *CmpCommon::diags() << DgSqlCode(-3242) << + DgString0(NAString("This DDL operation is not allowed in the specified catalog '" + qualObjName_.getCatalogName() + "'.")); + } else - *CmpCommon::diags() << DgSqlCode(-3242) << DgString0("DDL operations can only be done on trafodion or external tables."); + *CmpCommon::diags() << DgSqlCode(-3242) << DgString0("This DDL operation cannot be done."); + bindWA->setErrStatus(); return NULL; } @@ -4845,8 +4864,8 @@ RelExpr * ExeUtilCreateTableAs::bindNode(BindWA *bindWA) if ((NOT isVolatile_) && (NOT getTableName().isSeabase())) // can only create traf tables { - *CmpCommon::diags() << DgSqlCode(-3242) - << DgString0("DDL operations can only be done on trafodion or external tables."); + *CmpCommon::diags() << DgSqlCode(-3242) << + DgString0(NAString("This DDL operation is not allowed in the specified catalog '" + getTableName().getQualifiedNameObj().getCatalogName() + "'.")); bindWA->setErrStatus(); return NULL; @@ -5232,7 +5251,7 @@ RelExpr * ExeUtilFastDelete::bindNode(BindWA *bindWA) (naTable->isHiveTable())) { *CmpCommon::diags() << DgSqlCode(-3242) - << DgString0("Purgedata is not allowed for hive tables. Use Truncate command."); + << DgString0("Purgedata is not allowed for Hive tables. Use 'Truncate Table' command."); bindWA->setErrStatus(); return NULL; } http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/optimizer/RelExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExeUtil.h b/core/sql/optimizer/RelExeUtil.h index 73d6ed7..84ff343 100644 --- a/core/sql/optimizer/RelExeUtil.h +++ b/core/sql/optimizer/RelExeUtil.h @@ -146,6 +146,7 @@ public: //virtual ExprNode * getChild(long index); ExprNode * getExprNode(){return exprNode_;}; + const ExprNode * getExprNode() const {return exprNode_;}; //virtual void addLocalExpr(LIST(ExprNode *) &xlist, // LIST(NAString) &llist) const; @@ -261,6 +262,7 @@ public: virtual TrafDesc *createVirtualTableDesc(); ExprNode * getDDLNode(){return getExprNode();}; + const ExprNode * getDDLNode() const {return getExprNode();}; char * getDDLStmtText() { @@ -1151,7 +1153,6 @@ public: FROM_STRING, FROM_FILE }; - ExeUtilHiveQuery(const NAString &hive_query, HiveSourceType type, CollHeap *oHeap = CmpCommon::statementHeap()) @@ -1162,26 +1163,18 @@ public: type_(type), hiveQuery_(hive_query) { } - virtual NABoolean isExeUtilQueryType() { return TRUE; } - virtual RelExpr * copyTopNode(RelExpr *derivedNode = NULL, CollHeap* outHeap = 0); - virtual RelExpr * bindNode(BindWA *bindWAPtr); - - // method to do code generation virtual short codeGen(Generator*); - NAString &hiveQuery() { return hiveQuery_; } const NAString &hiveQuery() const { return hiveQuery_; } - HiveSourceType sourceType() { return type_;} private: HiveSourceType type_; NAString hiveQuery_; }; - class ExeUtilMaintainObject : public ExeUtilExpr { public: http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/optimizer/RelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExpr.cpp b/core/sql/optimizer/RelExpr.cpp index ea4495a..44e0e90 100644 --- a/core/sql/optimizer/RelExpr.cpp +++ b/core/sql/optimizer/RelExpr.cpp @@ -10263,7 +10263,6 @@ RelExpr *HbaseAccess::bindNode(BindWA *bindWA) return this; } - // CorrName &corrName = (CorrName&)getCorrName(); CorrName &corrName = getTableName(); NATable * naTable = NULL; @@ -10274,7 +10273,8 @@ RelExpr *HbaseAccess::bindNode(BindWA *bindWA) { *CmpCommon::diags() << DgSqlCode(-1388) - << DgTableName(corrName.getExposedNameAsAnsiString()); + << DgString0("Object") + << DgString1(corrName.getExposedNameAsAnsiString()); bindWA->setErrStatus(); return this; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/optimizer/hiveHook.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/hiveHook.h b/core/sql/optimizer/hiveHook.h index ead62c9..0376e8a 100644 --- a/core/sql/optimizer/hiveHook.h +++ b/core/sql/optimizer/hiveHook.h @@ -219,6 +219,22 @@ struct hive_tbl_desc return FALSE; } + NABoolean isExternalTable() + { + if (tableType_ && (strcmp(tableType_, "EXTERNAL_TABLE") == 0)) + return TRUE; + else + return FALSE; + } + + NABoolean isManagedTable() + { + if (tableType_ && (strcmp(tableType_, "MANAGED_TABLE") == 0)) + return TRUE; + else + return FALSE; + } + struct hive_sd_desc* getSDs() { return sd_; }; struct hive_skey_desc* getSortKeys(); http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/AllStmtDDL.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/AllStmtDDL.h b/core/sql/parser/AllStmtDDL.h index d966f78..5e6fd70 100644 --- a/core/sql/parser/AllStmtDDL.h +++ b/core/sql/parser/AllStmtDDL.h @@ -66,3 +66,5 @@ #include "StmtDDLCreateRole.h" #include "StmtDDLRoleGrant.h" #include "StmtDDLCleanupObjects.h" +#include "StmtDDLonHiveObjects.h" + http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/ElemDDLLikeOptions.cpp ---------------------------------------------------------------------- diff --git a/core/sql/parser/ElemDDLLikeOptions.cpp b/core/sql/parser/ElemDDLLikeOptions.cpp index fa40bfd..a252ffc 100644 --- a/core/sql/parser/ElemDDLLikeOptions.cpp +++ b/core/sql/parser/ElemDDLLikeOptions.cpp @@ -297,6 +297,36 @@ const NAString ElemDDLLikeOptWithoutLobColumns::getText() const return "ElemDDLLikeOptWithoutLobColumns"; } +// ----------------------------------------------------------------------- +// methods for class ElemDDLLikeOptWithHiveOptions +// ----------------------------------------------------------------------- + +ElemDDLLikeOptWithHiveOptions::ElemDDLLikeOptWithHiveOptions(NAString &hiveOptionsText) + : ElemDDLLikeOpt(ELM_LIKE_OPT_WITH_HIVE_OPTIONS), + hiveOptionsText_(hiveOptionsText) +{ +} + +// virtual destructor +ElemDDLLikeOptWithHiveOptions::~ElemDDLLikeOptWithHiveOptions() +{ +} + +// casting +ElemDDLLikeOptWithHiveOptions * ElemDDLLikeOptWithHiveOptions::castToElemDDLLikeOptWithHiveOptions() +{ + return this; +} + +// +// methods for tracing +// + +const NAString ElemDDLLikeOptWithHiveOptions::getText() const +{ + return "ElemDDLLikeOptWithHiveOptions"; +} + // // End of File http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/ElemDDLLikeOptions.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/ElemDDLLikeOptions.h b/core/sql/parser/ElemDDLLikeOptions.h index ef06865..8288385 100644 --- a/core/sql/parser/ElemDDLLikeOptions.h +++ b/core/sql/parser/ElemDDLLikeOptions.h @@ -348,4 +348,37 @@ private: }; // class ElemDDLLikeOptWithoutLobColumns +// ----------------------------------------------------------------------- +// definition of class ElemDDLLikeOptWithHiveOptions +// These are options that are specified during create through the use of +// 'with hive options' syntax. +// These are hive specific options that are passed to hive layer during +// creation of table. These are not interpreted by traf layer. +// +// create external hive table <htab> like <traf-table> +// with hive options 'hive-options' +// ----------------------------------------------------------------------- +class ElemDDLLikeOptWithHiveOptions : public ElemDDLLikeOpt +{ + +public: + + // constructor + ElemDDLLikeOptWithHiveOptions(NAString &hiveOptionsText); + + // virtual destructor + virtual ~ElemDDLLikeOptWithHiveOptions(); + + // cast + virtual ElemDDLLikeOptWithHiveOptions * castToElemDDLLikeOptWithHiveOptions(); + + // method for tracing + virtual const NAString getText() const; + + const NAString &getHiveOptionsText() const { return hiveOptionsText_;} + +private: + NAString hiveOptionsText_; +}; // class ElemDDLLikeOptWithHiveOptions + #endif // ELEMDDLLIKEOPTIONS_H http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/ElemDDLNode.cpp ---------------------------------------------------------------------- diff --git a/core/sql/parser/ElemDDLNode.cpp b/core/sql/parser/ElemDDLNode.cpp index 4d67199..c324be3 100644 --- a/core/sql/parser/ElemDDLNode.cpp +++ b/core/sql/parser/ElemDDLNode.cpp @@ -1858,6 +1858,12 @@ ElemDDLNode::castToStmtDDLRoleGrant() return NULL; } +StmtDDLonHiveObjects * +ElemDDLNode::castToStmtDDLonHiveObjects() +{ + return NULL; +} + // // accessors // http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/ElemDDLNode.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/ElemDDLNode.h b/core/sql/parser/ElemDDLNode.h index fb10591..fee5408 100644 --- a/core/sql/parser/ElemDDLNode.h +++ b/core/sql/parser/ElemDDLNode.h @@ -140,6 +140,7 @@ class ElemDDLLikeOptWithoutDivision; class ElemDDLLikeLimitColumnLength; class ElemDDLLikeOptWithoutRowFormat; class ElemDDLLikeOptWithoutLobColumns; +class ElemDDLLikeOptWithHiveOptions; class ElemDDLList; class ElemDDLLocation; class ElemDDLOptionList; @@ -350,6 +351,7 @@ class StmtDDLRegOrUnregObject; class StmtDDLCreateRole; class StmtDDLRoleGrant; class StmtDDLCleanupObjects; +class StmtDDLonHiveObjects; class StmtDDLCommentOn; class QualifiedName; @@ -467,6 +469,8 @@ public: virtual ElemDDLLikeLimitColumnLength * castToElemDDLLikeLimitColumnLength(); virtual ElemDDLLikeOptWithoutRowFormat* castToElemDDLLikeOptWithoutRowFormat(); virtual ElemDDLLikeOptWithoutLobColumns * castToElemDDLLikeOptWithoutLobColumns(); + virtual ElemDDLLikeOptWithHiveOptions * castToElemDDLLikeOptWithHiveOptions() + {return NULL;} virtual ElemDDLList * castToElemDDLList(); virtual ElemDDLLocation * castToElemDDLLocation(); virtual ElemDDLOptionList * castToElemDDLOptionList(); @@ -692,7 +696,7 @@ public: virtual StmtDDLRoleGrant * castToStmtDDLRoleGrant(); virtual StmtDDLCleanupObjects * castToStmtDDLCleanupObjects(); virtual StmtDDLCommentOn * castToStmtDDLCommentOn(); - + virtual StmtDDLonHiveObjects * castToStmtDDLonHiveObjects(); // // operator http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/ParDDLLikeOpts.cpp ---------------------------------------------------------------------- diff --git a/core/sql/parser/ParDDLLikeOpts.cpp b/core/sql/parser/ParDDLLikeOpts.cpp index c331b6b..b7b4ca4 100644 --- a/core/sql/parser/ParDDLLikeOpts.cpp +++ b/core/sql/parser/ParDDLLikeOpts.cpp @@ -116,6 +116,7 @@ ParDDLLikeOptsCreateTable::operator=( isLikeOptLimitColumnLengthSpec_ = likeOptions.isLikeOptLimitColumnLengthSpec_; isLikeOptWithoutRowFormatSpec_ = likeOptions.isLikeOptWithoutRowFormatSpec_; isLikeOptWithoutLobColumnsSpec_ = likeOptions.isLikeOptWithoutLobColumnsSpec_; + isLikeOptWithHiveOptionsSpec_ = likeOptions.isLikeOptWithHiveOptionsSpec_; isLikeOptWithComments_ = likeOptions.isLikeOptWithComments_; isLikeOptWithoutConstraints_ = likeOptions.isLikeOptWithoutConstraints_; @@ -127,6 +128,7 @@ ParDDLLikeOptsCreateTable::operator=( isLikeOptColumnLengthLimit_ = likeOptions.isLikeOptColumnLengthLimit_; isLikeOptWithoutRowFormat_ = likeOptions.isLikeOptWithoutRowFormat_; isLikeOptWithoutLobColumns_ = likeOptions.isLikeOptWithoutLobColumns_; + likeOptHiveOptions_ = likeOptions.likeOptHiveOptions_; if (this != &likeOptions) // make sure not assigning to self { @@ -164,6 +166,7 @@ ParDDLLikeOptsCreateTable::initializeDataMembers() isLikeOptLimitColumnLengthSpec_ = FALSE; isLikeOptWithoutRowFormatSpec_ = FALSE; isLikeOptWithoutLobColumnsSpec_ = FALSE; + isLikeOptWithHiveOptionsSpec_ = FALSE; isLikeOptWithComments_ = FALSE; isLikeOptWithoutConstraints_ = FALSE; @@ -313,6 +316,18 @@ ParDDLLikeOptsCreateTable::setLikeOption(ElemDDLLikeOpt * pLikeOption) isLikeOptWithoutLobColumnsSpec_ = TRUE; break; + case ELM_LIKE_OPT_WITH_HIVE_OPTIONS : + if (isLikeOptWithHiveOptionsSpec_) + { + // ERROR[3152] Duplicate WITH HIVE OPTIONS phrases were specified + // in LIKE clause in CREATE TABLE statement. + *SqlParser_Diags << DgSqlCode(-3152) << DgString0("HIVE OPTIONS"); + } + ComASSERT(pLikeOption->castToElemDDLLikeOptWithHiveOptions() != NULL); + likeOptHiveOptions_ = pLikeOption->castToElemDDLLikeOptWithHiveOptions()->getHiveOptionsText(); + isLikeOptWithHiveOptionsSpec_ = TRUE; + break; + default : NAAbort("ParDDLLikeOpts.C", __LINE__, "internal logic error"); break; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/ParDDLLikeOptsCreateTable.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/ParDDLLikeOptsCreateTable.h b/core/sql/parser/ParDDLLikeOptsCreateTable.h index 844696d..03d089a 100644 --- a/core/sql/parser/ParDDLLikeOptsCreateTable.h +++ b/core/sql/parser/ParDDLLikeOptsCreateTable.h @@ -149,6 +149,12 @@ public: return isLikeOptWithoutLobColumns_; } + const NAString & + getLikeOptHiveOptions() const + { + return likeOptHiveOptions_; + } + // mutators void setLikeOption(ElemDDLLikeOpt * pLikeOptParseNode); @@ -213,6 +219,12 @@ public: isLikeOptWithoutLobColumns_ = setting; } + void + setLikeHiveOptions(const NAString &opts) + { + likeOptHiveOptions_ = opts; + } + private: // --------------------------------------------------------------------- @@ -240,6 +252,7 @@ private: NABoolean isLikeOptLimitColumnLengthSpec_; NABoolean isLikeOptWithoutRowFormatSpec_; NABoolean isLikeOptWithoutLobColumnsSpec_; + NABoolean isLikeOptWithHiveOptionsSpec_; // legal Like options in DDL Create Table statements @@ -254,6 +267,7 @@ private: UInt32 isLikeOptColumnLengthLimit_; // in bytes; max UInt32 if no limit specified NABoolean isLikeOptWithoutRowFormat_; NABoolean isLikeOptWithoutLobColumns_; + NAString likeOptHiveOptions_; }; // class ParDDLLikeOptsCreateTable http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/SqlParserAux.cpp ---------------------------------------------------------------------- diff --git a/core/sql/parser/SqlParserAux.cpp b/core/sql/parser/SqlParserAux.cpp index 02f595c..094e5e8 100644 --- a/core/sql/parser/SqlParserAux.cpp +++ b/core/sql/parser/SqlParserAux.cpp @@ -182,7 +182,7 @@ void yyerror(const char *errtext) *SqlParser_Diags << DgSqlCode(-SQLCI_SYNTAX_ERROR); // Point to the end of offending token, // knowing that the Lexer has looked ahead by 2 characters - Int32 pos = SqlParser_CurrentParser->getLexer()->getInputPos(); + Int32 pos = SqlParser_CurrentParser->getLexer()->getInputPos(); StoreSyntaxError(inputStr,pos,*SqlParser_Diags,0, CharInfo::UTF8 ); @@ -1655,6 +1655,71 @@ QualifiedName * qualifiedNameFromStrings(ShortStringSequence *names) return result; } +SchemaName * schemaNameFromStrings(ShortStringSequence *names) +{ + assert(names); + UInt32 index = names->numParts(); + assert(index>0); + + NAString schName; + NAString catName; + + getNamePart(schName,names,index); + getNamePart(catName,names,index); + + if (index) { + // ~String0 is an invalid qualified name + *SqlParser_Diags << DgSqlCode(-3011) + << DgWString0(badNameFromStrings(names)); + return NULL; + } + + StringPos startPos = names->getPosition(); + delete names; + SchemaName *result = new (PARSERHEAP()) + SchemaName(schName, catName, PARSERHEAP()); + ComASSERT(result); + result->setNamePosition(startPos, CharHereIsaDoubleQuote(startPos)); + return result; +} + +// This method detects that a hive object is being processed and it +// sets the field foundHiveDDL_ in passed in hiveDDLInfo object. +// This field is later used to process hive DDL. +short preprocessHiveDDL(const NAString &catalogName, + Parser::HiveDDLInfo *hiveDDLInfo) +{ + // In some cases, parser is called to process statements. + // For ex, binder calls parser during view expansion in dml queries. + // Or internal MD definitions are processed during process startup. + // In these internal cases, skip special hive ddl processing + // Flag disableDDLcheck _ will be set in these cases. + if ((NOT hiveDDLInfo->disableDDLcheck_) && + (hiveDDLInfo->checkForDDL_) && + (NOT hiveDDLInfo->foundDDL_)) + { + hiveDDLInfo->checkForDDL_ = FALSE; + if (((NOT catalogName.isNull()) && + (catalogName.compareTo + (HIVE_SYSTEM_CATALOG) == 0)) || + ((catalogName.isNull()) && + (CmpCommon::getDefaultString(CATALOG).compareTo + (HIVE_SYSTEM_CATALOG) == 0))) + { + hiveDDLInfo->foundDDL_ = TRUE; + hiveDDLInfo->userSpecifiedStmt_ = + NAString(SqlParser_CurrentParser->inputStr(), + SqlParser_CurrentParser->inputStrLen()); + } + else if (hiveDDLInfo->backquotedDelimFound_) + { + yyerror(""); // emit syntax error + } + } + + return 0; +} + // if the schemaName part inName contains volatile schema prefix, then // return an error. Don't do this if volatile schema prefix is allowed // for internal queries. @@ -3311,6 +3376,14 @@ TableTokens::setTableTokens(StmtDDLCreateTable *pNode) pNode->setInsertMode(COM_MULTISET_TABLE_INSERT_MODE); break; + case TableTokens::TYPE_EXTERNAL_HIVE_TABLE: + pNode->setExternalHiveTable(TRUE); + break; + + case TableTokens::TYPE_MANAGED_HIVE_TABLE: + pNode->setManagedHiveTable(TRUE); + break; + default: NAAbort("TableTokens - TypeAttr", __LINE__, "internal logic error"); break; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/SqlParserAux.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/SqlParserAux.h b/core/sql/parser/SqlParserAux.h index fddfec3..7f6e635 100644 --- a/core/sql/parser/SqlParserAux.h +++ b/core/sql/parser/SqlParserAux.h @@ -407,10 +407,14 @@ NAWString badNameFromStrings(ShortStringSequence *names); QualifiedName * qualifiedNameFromStrings(ShortStringSequence *names); +SchemaName * schemaNameFromStrings(ShortStringSequence *names); + CorrName * corrNameFromStrings(ShortStringSequence *names); ColRefName *colRefNameFromStrings(ShortStringSequence *names); +short preprocessHiveDDL(const NAString &catalogName, + Parser::HiveDDLInfo *hiveDDLInfo); // The purpose of this function is to convert NAStrings that contain // delimited identifiers as detected by SqlLexer @@ -767,7 +771,15 @@ public: TYPE_VOLATILE_TABLE_MODE_SPECIAL1, TYPE_VOLATILE_SET_TABLE, TYPE_VOLATILE_MULTISET_TABLE, - TYPE_GHOST_TABLE + TYPE_GHOST_TABLE, + + // create external hive table. + // usage: create external hive table hive.hive.tgt like trafodion.sch.src; + TYPE_EXTERNAL_HIVE_TABLE, + + // create managed hive table. + // usage: create hive table hive.hive.tgt like trafodion.sch.src; + TYPE_MANAGED_HIVE_TABLE }; // load/in memory options http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/StmtDDLCleanupObjects.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/StmtDDLCleanupObjects.h b/core/sql/parser/StmtDDLCleanupObjects.h index 33820e9..633e618 100644 --- a/core/sql/parser/StmtDDLCleanupObjects.h +++ b/core/sql/parser/StmtDDLCleanupObjects.h @@ -68,8 +68,8 @@ public: SEQUENCE_, SCHEMA_PRIVATE_, SCHEMA_SHARED_, - HIVE_TABLE_, - HIVE_VIEW_, + // HIVE_TABLE_, + // HIVE_VIEW_, HBASE_TABLE_, UNKNOWN_, OBSOLETE_ http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/StmtDDLCreate.cpp ---------------------------------------------------------------------- diff --git a/core/sql/parser/StmtDDLCreate.cpp b/core/sql/parser/StmtDDLCreate.cpp index 460c16b..1a2e7f3 100644 --- a/core/sql/parser/StmtDDLCreate.cpp +++ b/core/sql/parser/StmtDDLCreate.cpp @@ -3821,7 +3821,9 @@ StmtDDLCreateTable::StmtDDLCreateTable(const QualifiedName & aTableQualName, pSGOptions_(NULL), createIfNotExists_(FALSE), mapToHbaseTable_(FALSE), - hbaseDataFormat_(FALSE) + hbaseDataFormat_(FALSE), + externalHiveTable_(FALSE), + managedHiveTable_(FALSE) { setChild(INDEX_TABLE_DEFINITION, pTableDefBody); setChild(INDEX_ATTRIBUTE_LIST, pCreateTableAttrList); @@ -6834,7 +6836,8 @@ StmtDDLCreateView::StmtDDLCreateView(const QualifiedName & viewQualName, pWithCheckOption_(optionalWithCheckOption), columnDefArray_(heap), viewUsages_(heap), - udfList_(heap) + udfList_(heap), + createIfNotExists_(FALSE) { setChild(INDEX_VIEW_OWNER, pOwner); } http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/StmtDDLCreateTable.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/StmtDDLCreateTable.h b/core/sql/parser/StmtDDLCreateTable.h index 43e52da..e25f5db 100644 --- a/core/sql/parser/StmtDDLCreateTable.h +++ b/core/sql/parser/StmtDDLCreateTable.h @@ -466,6 +466,12 @@ public: inline void setInsertMode(ComInsertMode insertMode); + void setExternalHiveTable(ComBoolean v) { externalHiveTable_ = v; } + NABoolean externalHiveTable() { return externalHiveTable_; } + + void setManagedHiveTable(ComBoolean v) { managedHiveTable_ = v; } + NABoolean managedHiveTable() { return managedHiveTable_; } + // sets the Constraint data member with the information in // the parse node pointed by pConstraint. This parse node // represents either a column or a table constraint definition. @@ -856,6 +862,14 @@ private: // create only if table doesnt exist. Otherwise just return. NABoolean createIfNotExists_; + // Usage: + // create external hive table hive.hive.tgt like trafodion.sch.src; + NABoolean externalHiveTable_; + + // Usage: + // create hive table hive.hive.tgt like trafodion.sch.src; + NABoolean managedHiveTable_; + }; // class StmtDDLCreateTable // ----------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/StmtDDLCreateView.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/StmtDDLCreateView.h b/core/sql/parser/StmtDDLCreateView.h index cc1734c..b2163cc 100644 --- a/core/sql/parser/StmtDDLCreateView.h +++ b/core/sql/parser/StmtDDLCreateView.h @@ -737,6 +737,9 @@ public: // first character) of the statement (within the // input string) + const NABoolean createIfNotExists() const { return createIfNotExists_; } + void setCreateIfNotExists(NABoolean v) { createIfNotExists_ = v; } + // // method for binding // @@ -838,6 +841,10 @@ private: ElemDDLNode * pWithCheckOption_; ElemDDLGrantee * pOwner_; + + // create if view does not exist. Otherwise just return. + NABoolean createIfNotExists_; + // All UDF's referenced in a statement // -------------------------------------------------------------------- LIST(OptUDFInfo *) udfList_; http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/StmtDDLDrop.cpp ---------------------------------------------------------------------- diff --git a/core/sql/parser/StmtDDLDrop.cpp b/core/sql/parser/StmtDDLDrop.cpp index 02e564e..5f47958 100644 --- a/core/sql/parser/StmtDDLDrop.cpp +++ b/core/sql/parser/StmtDDLDrop.cpp @@ -906,7 +906,8 @@ StmtDDLDropView::StmtDDLDropView(const QualifiedName & viewQualName, dropBehavior_(dropBehavior), isCleanupSpec_(cleanupSpec), isValidateSpec_(validateSpec), - pLogFile_(pLogFile) + pLogFile_(pLogFile), + dropIfExists_(FALSE) { } http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/StmtDDLDropView.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/StmtDDLDropView.h b/core/sql/parser/StmtDDLDropView.h index d82358b..d805afe 100644 --- a/core/sql/parser/StmtDDLDropView.h +++ b/core/sql/parser/StmtDDLDropView.h @@ -81,6 +81,9 @@ public: inline const NABoolean isLogFileSpecified() const; inline const NAString & getLogFile() const; + const NABoolean dropIfExists() const { return dropIfExists_; } + void setDropIfExists(NABoolean v) { dropIfExists_ = v; } + // for binding ExprNode * bindNode(BindWA *bindWAPtr); @@ -98,6 +101,9 @@ private: NABoolean isValidateSpec_; NAString *pLogFile_; + // drop only if view exists. Otherwise just return. + NABoolean dropIfExists_; + }; // class StmtDDLDropView // ----------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/StmtDDLMisc.cpp ---------------------------------------------------------------------- diff --git a/core/sql/parser/StmtDDLMisc.cpp b/core/sql/parser/StmtDDLMisc.cpp index af2c9d4..c98984b 100644 --- a/core/sql/parser/StmtDDLMisc.cpp +++ b/core/sql/parser/StmtDDLMisc.cpp @@ -97,8 +97,6 @@ StmtDDLCleanupObjects::bindNode(BindWA * pBindWA) (type_ == VIEW_) || (type_ == SCHEMA_PRIVATE_) || (type_ == SCHEMA_SHARED_) || - (type_ == HIVE_TABLE_) || - (type_ == HIVE_VIEW_) || (type_ == HBASE_TABLE_) || (type_ == UNKNOWN_)) { http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/StmtDDLRegOrUnregHive.cpp ---------------------------------------------------------------------- diff --git a/core/sql/parser/StmtDDLRegOrUnregHive.cpp b/core/sql/parser/StmtDDLRegOrUnregHive.cpp index 1a49e94..10af8b7 100644 --- a/core/sql/parser/StmtDDLRegOrUnregHive.cpp +++ b/core/sql/parser/StmtDDLRegOrUnregHive.cpp @@ -59,7 +59,7 @@ StmtDDLRegOrUnregObject::StmtDDLRegOrUnregObject(const QualifiedName & origObjNa const StorageType storageType, const NABoolean isRegister, const ComObjectType objType, - const NABoolean existsOption, + const NABoolean registeredOption, const NABoolean isInternal, const NABoolean cascade, const NABoolean cleanup, @@ -70,7 +70,7 @@ StmtDDLRegOrUnregObject::StmtDDLRegOrUnregObject(const QualifiedName & origObjNa storageType_(storageType), isRegister_(isRegister), objType_(objType), - existsOption_(existsOption), + registeredOption_(registeredOption), isInternal_(isInternal), cascade_(cascade), cleanup_(cleanup) http://git-wip-us.apache.org/repos/asf/trafodion/blob/6f490daf/core/sql/parser/StmtDDLRegOrUnregHive.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/StmtDDLRegOrUnregHive.h b/core/sql/parser/StmtDDLRegOrUnregHive.h index a32f249..ee3b4f8 100644 --- a/core/sql/parser/StmtDDLRegOrUnregHive.h +++ b/core/sql/parser/StmtDDLRegOrUnregHive.h @@ -71,7 +71,7 @@ public: // true, register. false, unregister const NABoolean isRegister, const ComObjectType objType, - const NABoolean existsOption, + const NABoolean registeredOption, const NABoolean isInternal, const NABoolean cascade, const NABoolean cleanup, @@ -98,7 +98,7 @@ public: {return objQualName_;} const NABoolean &isRegister() const { return isRegister_; } const ComObjectType &objType() const { return objType_; } - const NABoolean &existsOption() const { return existsOption_; } + const NABoolean ®isteredOption() const { return registeredOption_; } const NABoolean &isInternal() const { return isInternal_; } @@ -130,7 +130,7 @@ private: // For unregister operation: // true, unregister if registered. // false, return error if not registered. - NABoolean existsOption_; + NABoolean registeredOption_; // true if this object was registered internally by trafodion NABoolean isInternal_;
