Repository: trafodion Updated Branches: refs/heads/master 0db94eaf7 -> 0e7d16dc9
CTAS fixes for ddl on hive objects Project: http://git-wip-us.apache.org/repos/asf/trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/trafodion/commit/bb3dee8f Tree: http://git-wip-us.apache.org/repos/asf/trafodion/tree/bb3dee8f Diff: http://git-wip-us.apache.org/repos/asf/trafodion/diff/bb3dee8f Branch: refs/heads/master Commit: bb3dee8fdd0d0593073dd39162c381e0913ba734 Parents: 8001c15 Author: Anoop Sharma <[email protected]> Authored: Sat Jun 16 19:17:06 2018 +0000 Committer: Anoop Sharma <[email protected]> Committed: Sat Jun 16 19:17:06 2018 +0000 ---------------------------------------------------------------------- core/sql/common/OperTypeEnum.h | 1 - core/sql/generator/GenExplain.cpp | 35 +++ core/sql/optimizer/RelExeUtil.cpp | 51 +++- core/sql/optimizer/RelExeUtil.h | 4 + core/sql/parser/ElemDDLLikeOptions.cpp | 31 --- core/sql/parser/ElemDDLLikeOptions.h | 33 --- core/sql/parser/ElemDDLNode.h | 3 - core/sql/parser/ParDDLLikeOpts.cpp | 14 -- core/sql/parser/ParDDLLikeOptsCreateTable.h | 1 - core/sql/parser/SqlParserAux.cpp | 8 - core/sql/parser/SqlParserAux.h | 8 - core/sql/parser/StmtDDLCreate.cpp | 4 +- core/sql/parser/StmtDDLCreateTable.h | 20 +- core/sql/parser/StmtDDLonHiveObjects.h | 2 - core/sql/parser/sqlparser.y | 182 ++++++-------- core/sql/regress/hive/DIFF008.KNOWN | 2 +- core/sql/regress/hive/EXPECTED008 | 287 ++++++++++++++++++----- core/sql/regress/hive/FILTER008 | 1 + core/sql/regress/hive/TEST008 | 73 ++++-- core/sql/sqlcomp/CmpDescribe.cpp | 92 -------- core/sql/sqlcomp/CmpDescribe.h | 8 - core/sql/sqlcomp/CmpSeabaseDDL.h | 5 - core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 139 +---------- core/sql/sqlcomp/DefaultConstants.h | 11 +- core/sql/sqlcomp/nadefaults.cpp | 2 + core/sql/sqlcomp/parser.cpp | 18 +- core/sql/sqlcomp/parser.h | 5 + 27 files changed, 459 insertions(+), 581 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/common/OperTypeEnum.h ---------------------------------------------------------------------- diff --git a/core/sql/common/OperTypeEnum.h b/core/sql/common/OperTypeEnum.h index 135264c..e375c0a 100644 --- a/core/sql/common/OperTypeEnum.h +++ b/core/sql/common/OperTypeEnum.h @@ -1096,7 +1096,6 @@ enum OperatorTypeEnum { ELM_LIKE_OPT_LIMIT_COLUMN_LENGTH, ELM_LIKE_OPT_WITHOUT_ROW_FORMAT_ELEM, ELM_LIKE_OPT_WITHOUT_LOB_COLUMNS, - ELM_LIKE_OPT_WITH_HIVE_OPTIONS, ELM_LOCATION_ELEM, ELM_OPTION_LIST, ELM_PARALLEL_EXEC_ELEM, http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/generator/GenExplain.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenExplain.cpp b/core/sql/generator/GenExplain.cpp index 6234952..11a5a55 100644 --- a/core/sql/generator/GenExplain.cpp +++ b/core/sql/generator/GenExplain.cpp @@ -2124,6 +2124,41 @@ ExplainTuple * ExeUtilWnrInsert::addSpecificExplainInfo( return(explainTuple); } +ExplainTuple * ExeUtilCreateTableAs::addSpecificExplainInfo( + ExplainTupleMaster *explainTuple, + ComTdb *tdb, + Generator *generator) +{ + + Lng32 maxBufLen = 2000; + maxBufLen = MAXOF(maxBufLen, ctQuery_.length()); + maxBufLen = MAXOF(maxBufLen, siQuery_.length()); + maxBufLen = MAXOF(maxBufLen, viQuery_.length()); + maxBufLen = MAXOF(maxBufLen, usQuery_.length()); + + maxBufLen = MINOF(maxBufLen, 4000); + maxBufLen++; + + char buf[maxBufLen]; + snprintf(buf, maxBufLen, "CreateQuery: %s ", + (ctQuery_.length() > 0 ? ctQuery_.data() : "NULL")); + explainTuple->setDescription(buf); + + snprintf(buf, maxBufLen, "InsertQuery: %s ", + (viQuery_.length() > 0 ? viQuery_.data() : "NULL")); + explainTuple->setDescription(buf); + + snprintf(buf, maxBufLen, "UpsertLoadQuery: %s ", + (siQuery_.length() > 0 ? siQuery_.data() : "NULL")); + explainTuple->setDescription(buf); + + snprintf(buf, maxBufLen, "UpdStatsQuery: %s ", + (usQuery_.length() > 0 ? usQuery_.data() : "NULL")); + explainTuple->setDescription(buf); + + return(explainTuple); +} + const char * ExplainFunc::getVirtualTableName() //{ return "EXPLAIN__"; } {return getVirtualTableNameStr();} http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/optimizer/RelExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExeUtil.cpp b/core/sql/optimizer/RelExeUtil.cpp index 7b96463..971a537 100644 --- a/core/sql/optimizer/RelExeUtil.cpp +++ b/core/sql/optimizer/RelExeUtil.cpp @@ -4864,8 +4864,17 @@ RelExpr * ExeUtilCreateTableAs::bindNode(BindWA *bindWA) if (bindWA->errStatus()) return NULL; + NABoolean isHive = FALSE; + if ((getTableName().isHive()) && + (CmpCommon::getDefault(TRAF_DDL_ON_HIVE_OBJECTS) == DF_ON)) + { + isHive = TRUE; + upsertUsingLoadAllowed = FALSE; + } + if ((NOT isVolatile_) && - (NOT getTableName().isSeabase())) // can only create traf tables + (NOT getTableName().isSeabase()) && // can only create traf tables + (NOT isHive)) { *CmpCommon::diags() << DgSqlCode(-3242) << DgString0(NAString("This DDL operation is not allowed in the specified catalog '" + getTableName().getQualifiedNameObj().getCatalogName() + "'.")); @@ -4943,7 +4952,9 @@ RelExpr * ExeUtilCreateTableAs::bindNode(BindWA *bindWA) } else if (createTableNode->isVolatile()) ctQuery_ = "CREATE VOLATILE TABLE "; - else + else if ((isHive) && (createTableNode->isExternal())) + ctQuery_ = "CREATE EXTERNAL TABLE "; + else ctQuery_ = "CREATE TABLE "; if (createTableNode->createIfNotExists()) @@ -4970,6 +4981,7 @@ RelExpr * ExeUtilCreateTableAs::bindNode(BindWA *bindWA) ctQuery_ += "( "; + NAString hiveType; if (! pTableDefBody) { for (CollIndex i = 0; i < retDesc->getDegree(); i++) @@ -4993,7 +5005,13 @@ RelExpr * ExeUtilCreateTableAs::bindNode(BindWA *bindWA) colDef += " "; NAType &colType = (NAType&)(queryRoot->compExpr()[i].getType()); - colType.getMyTypeAsText(&colDef); + if (isHive) + { + colType.getMyTypeAsHiveText(&hiveType); + colDef += hiveType; + } + else + colType.getMyTypeAsText(&colDef); if (colType.isLob()) upsertUsingLoadAllowed = FALSE; @@ -5138,10 +5156,10 @@ RelExpr * ExeUtilCreateTableAs::bindNode(BindWA *bindWA) // if attribute list is specified, append that to col definition. if (createTableNode->getChild(1/*StmtDDLCreateTable::INDEX_ATTRIBUTE_LIST*/)) { - ctQuery_ += " "; - ctQuery_.append(&stmtText[attrListStartPos], - attrListEndPos - attrListStartPos); - } + ctQuery_ += " "; + ctQuery_.append(&stmtText[attrListStartPos], + attrListEndPos - attrListStartPos); + } } else { @@ -5150,7 +5168,24 @@ RelExpr * ExeUtilCreateTableAs::bindNode(BindWA *bindWA) ctQuery_ = ""; ctQuery_.append(stmtText, attrListEndPos); } - + + if (NOT createTableNode->getHiveOptions().isNull()) + { + if (NOT isHive) + { + *CmpCommon::diags() << DgSqlCode(-3242) << + DgString0(NAString("WITH HIVE OPTIONS cannot be specified for non-Hive tables.")); + + bindWA->setErrStatus(); + return NULL; + } + else + { + ctQuery_ += " "; + ctQuery_ += createTableNode->getHiveOptions(); + } + } + if (createTableNode->isInMemoryObjectDefn()) ctQuery_.append(" IN MEMORY "); http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/optimizer/RelExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExeUtil.h b/core/sql/optimizer/RelExeUtil.h index 84ff343..d94ab63 100644 --- a/core/sql/optimizer/RelExeUtil.h +++ b/core/sql/optimizer/RelExeUtil.h @@ -983,6 +983,10 @@ public: // method to do code generation virtual short codeGen(Generator*); + ExplainTuple *addSpecificExplainInfo(ExplainTupleMaster *explainTuple, + ComTdb * tdb, + Generator *generator); + virtual NABoolean explainSupported() { return TRUE; } virtual NABoolean producesOutput() { http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/parser/ElemDDLLikeOptions.cpp ---------------------------------------------------------------------- diff --git a/core/sql/parser/ElemDDLLikeOptions.cpp b/core/sql/parser/ElemDDLLikeOptions.cpp index a252ffc..8e6d7e3 100644 --- a/core/sql/parser/ElemDDLLikeOptions.cpp +++ b/core/sql/parser/ElemDDLLikeOptions.cpp @@ -297,37 +297,6 @@ const NAString ElemDDLLikeOptWithoutLobColumns::getText() const return "ElemDDLLikeOptWithoutLobColumns"; } -// ----------------------------------------------------------------------- -// methods for class ElemDDLLikeOptWithHiveOptions -// ----------------------------------------------------------------------- - -ElemDDLLikeOptWithHiveOptions::ElemDDLLikeOptWithHiveOptions(NAString &hiveOptionsText) - : ElemDDLLikeOpt(ELM_LIKE_OPT_WITH_HIVE_OPTIONS), - hiveOptionsText_(hiveOptionsText) -{ -} - -// virtual destructor -ElemDDLLikeOptWithHiveOptions::~ElemDDLLikeOptWithHiveOptions() -{ -} - -// casting -ElemDDLLikeOptWithHiveOptions * ElemDDLLikeOptWithHiveOptions::castToElemDDLLikeOptWithHiveOptions() -{ - return this; -} - -// -// methods for tracing -// - -const NAString ElemDDLLikeOptWithHiveOptions::getText() const -{ - return "ElemDDLLikeOptWithHiveOptions"; -} - - // // End of File // http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/parser/ElemDDLLikeOptions.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/ElemDDLLikeOptions.h b/core/sql/parser/ElemDDLLikeOptions.h index 8288385..ef06865 100644 --- a/core/sql/parser/ElemDDLLikeOptions.h +++ b/core/sql/parser/ElemDDLLikeOptions.h @@ -348,37 +348,4 @@ private: }; // class ElemDDLLikeOptWithoutLobColumns -// ----------------------------------------------------------------------- -// definition of class ElemDDLLikeOptWithHiveOptions -// These are options that are specified during create through the use of -// 'with hive options' syntax. -// These are hive specific options that are passed to hive layer during -// creation of table. These are not interpreted by traf layer. -// -// create external hive table <htab> like <traf-table> -// with hive options 'hive-options' -// ----------------------------------------------------------------------- -class ElemDDLLikeOptWithHiveOptions : public ElemDDLLikeOpt -{ - -public: - - // constructor - ElemDDLLikeOptWithHiveOptions(NAString &hiveOptionsText); - - // virtual destructor - virtual ~ElemDDLLikeOptWithHiveOptions(); - - // cast - virtual ElemDDLLikeOptWithHiveOptions * castToElemDDLLikeOptWithHiveOptions(); - - // method for tracing - virtual const NAString getText() const; - - const NAString &getHiveOptionsText() const { return hiveOptionsText_;} - -private: - NAString hiveOptionsText_; -}; // class ElemDDLLikeOptWithHiveOptions - #endif // ELEMDDLLIKEOPTIONS_H http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/parser/ElemDDLNode.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/ElemDDLNode.h b/core/sql/parser/ElemDDLNode.h index fee5408..d00200f 100644 --- a/core/sql/parser/ElemDDLNode.h +++ b/core/sql/parser/ElemDDLNode.h @@ -140,7 +140,6 @@ class ElemDDLLikeOptWithoutDivision; class ElemDDLLikeLimitColumnLength; class ElemDDLLikeOptWithoutRowFormat; class ElemDDLLikeOptWithoutLobColumns; -class ElemDDLLikeOptWithHiveOptions; class ElemDDLList; class ElemDDLLocation; class ElemDDLOptionList; @@ -469,8 +468,6 @@ public: virtual ElemDDLLikeLimitColumnLength * castToElemDDLLikeLimitColumnLength(); virtual ElemDDLLikeOptWithoutRowFormat* castToElemDDLLikeOptWithoutRowFormat(); virtual ElemDDLLikeOptWithoutLobColumns * castToElemDDLLikeOptWithoutLobColumns(); - virtual ElemDDLLikeOptWithHiveOptions * castToElemDDLLikeOptWithHiveOptions() - {return NULL;} virtual ElemDDLList * castToElemDDLList(); virtual ElemDDLLocation * castToElemDDLLocation(); virtual ElemDDLOptionList * castToElemDDLOptionList(); http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/parser/ParDDLLikeOpts.cpp ---------------------------------------------------------------------- diff --git a/core/sql/parser/ParDDLLikeOpts.cpp b/core/sql/parser/ParDDLLikeOpts.cpp index b7b4ca4..fee16e2 100644 --- a/core/sql/parser/ParDDLLikeOpts.cpp +++ b/core/sql/parser/ParDDLLikeOpts.cpp @@ -116,7 +116,6 @@ ParDDLLikeOptsCreateTable::operator=( isLikeOptLimitColumnLengthSpec_ = likeOptions.isLikeOptLimitColumnLengthSpec_; isLikeOptWithoutRowFormatSpec_ = likeOptions.isLikeOptWithoutRowFormatSpec_; isLikeOptWithoutLobColumnsSpec_ = likeOptions.isLikeOptWithoutLobColumnsSpec_; - isLikeOptWithHiveOptionsSpec_ = likeOptions.isLikeOptWithHiveOptionsSpec_; isLikeOptWithComments_ = likeOptions.isLikeOptWithComments_; isLikeOptWithoutConstraints_ = likeOptions.isLikeOptWithoutConstraints_; @@ -166,7 +165,6 @@ ParDDLLikeOptsCreateTable::initializeDataMembers() isLikeOptLimitColumnLengthSpec_ = FALSE; isLikeOptWithoutRowFormatSpec_ = FALSE; isLikeOptWithoutLobColumnsSpec_ = FALSE; - isLikeOptWithHiveOptionsSpec_ = FALSE; isLikeOptWithComments_ = FALSE; isLikeOptWithoutConstraints_ = FALSE; @@ -316,18 +314,6 @@ ParDDLLikeOptsCreateTable::setLikeOption(ElemDDLLikeOpt * pLikeOption) isLikeOptWithoutLobColumnsSpec_ = TRUE; break; - case ELM_LIKE_OPT_WITH_HIVE_OPTIONS : - if (isLikeOptWithHiveOptionsSpec_) - { - // ERROR[3152] Duplicate WITH HIVE OPTIONS phrases were specified - // in LIKE clause in CREATE TABLE statement. - *SqlParser_Diags << DgSqlCode(-3152) << DgString0("HIVE OPTIONS"); - } - ComASSERT(pLikeOption->castToElemDDLLikeOptWithHiveOptions() != NULL); - likeOptHiveOptions_ = pLikeOption->castToElemDDLLikeOptWithHiveOptions()->getHiveOptionsText(); - isLikeOptWithHiveOptionsSpec_ = TRUE; - break; - default : NAAbort("ParDDLLikeOpts.C", __LINE__, "internal logic error"); break; http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/parser/ParDDLLikeOptsCreateTable.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/ParDDLLikeOptsCreateTable.h b/core/sql/parser/ParDDLLikeOptsCreateTable.h index 03d089a..0022ddf 100644 --- a/core/sql/parser/ParDDLLikeOptsCreateTable.h +++ b/core/sql/parser/ParDDLLikeOptsCreateTable.h @@ -252,7 +252,6 @@ private: NABoolean isLikeOptLimitColumnLengthSpec_; NABoolean isLikeOptWithoutRowFormatSpec_; NABoolean isLikeOptWithoutLobColumnsSpec_; - NABoolean isLikeOptWithHiveOptionsSpec_; // legal Like options in DDL Create Table statements http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/parser/SqlParserAux.cpp ---------------------------------------------------------------------- diff --git a/core/sql/parser/SqlParserAux.cpp b/core/sql/parser/SqlParserAux.cpp index 094e5e8..faf0103 100644 --- a/core/sql/parser/SqlParserAux.cpp +++ b/core/sql/parser/SqlParserAux.cpp @@ -3376,14 +3376,6 @@ TableTokens::setTableTokens(StmtDDLCreateTable *pNode) pNode->setInsertMode(COM_MULTISET_TABLE_INSERT_MODE); break; - case TableTokens::TYPE_EXTERNAL_HIVE_TABLE: - pNode->setExternalHiveTable(TRUE); - break; - - case TableTokens::TYPE_MANAGED_HIVE_TABLE: - pNode->setManagedHiveTable(TRUE); - break; - default: NAAbort("TableTokens - TypeAttr", __LINE__, "internal logic error"); break; http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/parser/SqlParserAux.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/SqlParserAux.h b/core/sql/parser/SqlParserAux.h index 7f6e635..e9ac2d4 100644 --- a/core/sql/parser/SqlParserAux.h +++ b/core/sql/parser/SqlParserAux.h @@ -772,14 +772,6 @@ public: TYPE_VOLATILE_SET_TABLE, TYPE_VOLATILE_MULTISET_TABLE, TYPE_GHOST_TABLE, - - // create external hive table. - // usage: create external hive table hive.hive.tgt like trafodion.sch.src; - TYPE_EXTERNAL_HIVE_TABLE, - - // create managed hive table. - // usage: create hive table hive.hive.tgt like trafodion.sch.src; - TYPE_MANAGED_HIVE_TABLE }; // load/in memory options http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/parser/StmtDDLCreate.cpp ---------------------------------------------------------------------- diff --git a/core/sql/parser/StmtDDLCreate.cpp b/core/sql/parser/StmtDDLCreate.cpp index 03cfbb9..fc9ddb7 100644 --- a/core/sql/parser/StmtDDLCreate.cpp +++ b/core/sql/parser/StmtDDLCreate.cpp @@ -3821,9 +3821,7 @@ StmtDDLCreateTable::StmtDDLCreateTable(const QualifiedName & aTableQualName, pSGOptions_(NULL), createIfNotExists_(FALSE), mapToHbaseTable_(FALSE), - hbaseDataFormat_(FALSE), - externalHiveTable_(FALSE), - managedHiveTable_(FALSE) + hbaseDataFormat_(FALSE) { setChild(INDEX_TABLE_DEFINITION, pTableDefBody); setChild(INDEX_ATTRIBUTE_LIST, pCreateTableAttrList); http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/parser/StmtDDLCreateTable.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/StmtDDLCreateTable.h b/core/sql/parser/StmtDDLCreateTable.h index e25f5db..559fbc1 100644 --- a/core/sql/parser/StmtDDLCreateTable.h +++ b/core/sql/parser/StmtDDLCreateTable.h @@ -394,6 +394,11 @@ public: // returns TURE if table feature clause appears; // returns FALSE otherwise + inline const NAString &getHiveOptions() + {return hiveOptions_; } + void setHiveOptions(NAString h) + {hiveOptions_ = h;} + // POS inline NABoolean isPOSNumPartnsSpecified() const; inline NABoolean isPOSInitialTableSizeSpecified() const; @@ -466,12 +471,6 @@ public: inline void setInsertMode(ComInsertMode insertMode); - void setExternalHiveTable(ComBoolean v) { externalHiveTable_ = v; } - NABoolean externalHiveTable() { return externalHiveTable_; } - - void setManagedHiveTable(ComBoolean v) { managedHiveTable_ = v; } - NABoolean managedHiveTable() { return managedHiveTable_; } - // sets the Constraint data member with the information in // the parse node pointed by pConstraint. This parse node // represents either a column or a table constraint definition. @@ -862,13 +861,8 @@ private: // create only if table doesnt exist. Otherwise just return. NABoolean createIfNotExists_; - // Usage: - // create external hive table hive.hive.tgt like trafodion.sch.src; - NABoolean externalHiveTable_; - - // Usage: - // create hive table hive.hive.tgt like trafodion.sch.src; - NABoolean managedHiveTable_; + // optional options specified during CTAS into a hive table + NAString hiveOptions_; }; // class StmtDDLCreateTable http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/parser/StmtDDLonHiveObjects.h ---------------------------------------------------------------------- diff --git a/core/sql/parser/StmtDDLonHiveObjects.h b/core/sql/parser/StmtDDLonHiveObjects.h index cd36f50..7ec1e5b 100644 --- a/core/sql/parser/StmtDDLonHiveObjects.h +++ b/core/sql/parser/StmtDDLonHiveObjects.h @@ -66,7 +66,6 @@ public: ALTER_, TRUNCATE_, PASSTHRU_DDL_, - CREATE_LIKE_TRAF_, UNKNOWN_OPER_ }; @@ -125,7 +124,6 @@ public: case ALTER_ : return "alter"; case TRUNCATE_ : return "truncate"; case PASSTHRU_DDL_ : return "passthru"; - case CREATE_LIKE_TRAF_: return "create_like_traf"; case UNKNOWN_OPER_ : return "unknown"; default : return "unknown"; } // switch http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/parser/sqlparser.y ---------------------------------------------------------------------- diff --git a/core/sql/parser/sqlparser.y b/core/sql/parser/sqlparser.y index 0fd04a1..9f58820 100755 --- a/core/sql/parser/sqlparser.y +++ b/core/sql/parser/sqlparser.y @@ -2925,6 +2925,8 @@ static void enableMakeQuotedStringISO88591Mechanism() %type <item> locking_stmt %type <parTriggerScopeType> optional_row_table +%type <stringval> optional_hive_options + %type <tableTokens> create_table_start_tokens %type <tableLoadAttrEnum> ctas_load_and_in_memory_options %type <pElemDDL> ctas_insert_columns @@ -25004,6 +25006,15 @@ udf_version_tag_clause : TOK_VERSION TOK_TAG std_char_string_literal delete $3; // std_char_string_literal } +optional_hive_options : empty + { + $$ = NULL; + } + | TOK_HIVE TOK_OPTIONS QUOTED_STRING + { + $$ = $3; + } + /* type pStmtDDL */ table_definition : create_table_start_tokens ddl_qualified_name @@ -25013,14 +25024,6 @@ table_definition : create_table_start_tokens optional_map_to_hbase_clause optional_hbase_data_format { - if (($1->getType() == TableTokens::TYPE_EXTERNAL_HIVE_TABLE) || - ($1->getType() == TableTokens::TYPE_MANAGED_HIVE_TABLE)) - { - *SqlParser_Diags << DgSqlCode(-3242) - << DgString0("LIKE clause must be specified to create this Hive table."); - YYERROR; - } - $1->setOptions(TableTokens::OPT_NONE); QualifiedName * qn; if ($1->isVolatile()) @@ -25056,10 +25059,9 @@ table_definition : create_table_start_tokens StmtDDLCreateTable *pNode = new (PARSERHEAP()) StmtDDLCreateTable( - *qn /*ddl_qualified_name*/, - $3 /*table_definition_body*/, - $4 /*optional_create_table_ - *attribute_list*/, + *qn, //ddl_qualified_name + $3, //table_definition_body + $4, //optional_create_table_attribute_list NULL, NULL, PARSERHEAP()); @@ -25084,12 +25086,13 @@ table_definition : create_table_start_tokens } $$ = pNode; - delete $1; /*TableTokens*/ - delete $2 /*ddl_qualified_name*/; + delete $1; //TableTokens + delete $2; //ddl_qualified_name } -table_definition : create_table_start_tokens ddl_qualified_name - like_definition +table_definition : create_table_start_tokens + ddl_qualified_name + like_definition { $1->setOptions(TableTokens::OPT_NONE); QualifiedName * qn; @@ -25100,12 +25103,11 @@ table_definition : create_table_start_tokens ddl_qualified_name if (! qn) YYABORT; - StmtDDLNode *rNode = NULL; StmtDDLCreateTable *pNode = new (PARSERHEAP()) StmtDDLCreateTable( - *qn /*ddl_qualified_name*/, - $3 /*like_definition*/, + *qn, //ddl_qualified_name + $3, //like_definition NULL, NULL, NULL, @@ -25120,34 +25122,9 @@ table_definition : create_table_start_tokens ddl_qualified_name ParNameCTLocListPtr = NULL; } - rNode = pNode; - if (($1->getType() == TableTokens::TYPE_EXTERNAL_HIVE_TABLE) || - ($1->getType() == TableTokens::TYPE_MANAGED_HIVE_TABLE)) - { - SqlParser_CurrentParser->hiveDDLInfo_-> - setValues(TRUE, StmtDDLonHiveObjects::CREATE_LIKE_TRAF_, - StmtDDLonHiveObjects::TABLE_, $1->ifNotExistsSet()); - - SqlParser_CurrentParser->hiveDDLInfo_->foundDDL_ = TRUE; - - NAString hiveDDL; - StmtDDLonHiveObjects *hNode = - new (PARSERHEAP()) - StmtDDLonHiveObjects( - StmtDDLonHiveObjects::CREATE_LIKE_TRAF_, - StmtDDLonHiveObjects::TABLE_, - $1->ifNotExistsSet(), - qn->getQualifiedNameAsAnsiString(), - hiveDDL, - PARSERHEAP()); - - hNode->setChild(0, pNode); - rNode = hNode; - } - - $$ = rNode; - delete $1; /*TableTokens*/ - delete $2 /*ddl_qualified_name*/; + $$ = pNode; + delete $1; //TableTokens + delete $2; //ddl_qualified_name } | TOK_CREATE special_table_name @@ -25178,13 +25155,18 @@ table_definition : create_table_start_tokens ddl_qualified_name } | create_table_start_tokens - ddl_qualified_name - table_definition_body - optional_create_table_attribute_list - create_table_as_attr_list_end + ddl_qualified_name + table_definition_body + optional_create_table_attribute_list + create_table_as_attr_list_end ctas_load_and_in_memory_options ctas_insert_columns + optional_hive_options create_table_as_token + { + if (CmpCommon::getDefault(HIVE_CTAS_IN_NATIVE_MODE) == DF_OFF) + SqlParser_CurrentParser->hiveDDLInfo_->setFoundDDL(FALSE); + } optional_locking_stmt_list query_expression optional_limit_spec @@ -25198,15 +25180,6 @@ table_definition : create_table_start_tokens ddl_qualified_name YYERROR; } - if (($1->getType() == TableTokens::TYPE_EXTERNAL_HIVE_TABLE) || - ($1->getType() == TableTokens::TYPE_MANAGED_HIVE_TABLE)) - { - // create hive table .. as.. not supported. - *SqlParser_Diags << DgSqlCode(-3242) - << DgString0("'create hive table ... as ...' construct is not allowed."); - YYERROR; - } - $1->setOptions($6); if ($1->isVolatile()) qn = processVolatileDDLName($2, FALSE, FALSE); @@ -25215,9 +25188,9 @@ table_definition : create_table_start_tokens ddl_qualified_name if (! qn) YYABORT; - RelRoot *top = finalize($10); + RelRoot *top = finalize($12); //limit clause - if ($11) + if ($13) { if (top->getFirstNRows() >= 0) { @@ -25227,16 +25200,16 @@ table_definition : create_table_start_tokens ddl_qualified_name else { NABoolean negate; - if ($11->castToConstValue(negate)) + if ($13->castToConstValue(negate)) { - ConstValue * limit = (ConstValue*)$11; + ConstValue * limit = (ConstValue*)$13; Lng32 scale = 0; top->setFirstNRows(limit->getExactNumericValue(scale)); top->setFirstNRowsParam(NULL); } else { - top->setFirstNRowsParam($11); + top->setFirstNRowsParam($13); top->setFirstNRows(-1); } } @@ -25245,11 +25218,10 @@ table_definition : create_table_start_tokens ddl_qualified_name StmtDDLCreateTable *pNode = new (PARSERHEAP()) StmtDDLCreateTable( - *qn /*ddl_qualified_name*/, - $3 /*table_definition_body*/, - $4 /*optional_create_table_ - *attribute_list*/, - $7, /* insert column list */ + *qn, //ddl_qualified_name + $3, // table_definition_body + $4, // optional_create_table_attribute_list + $7, // insert column list top, PARSERHEAP()); $1->setTableTokens(pNode); @@ -25266,18 +25238,26 @@ table_definition : create_table_start_tokens ddl_qualified_name delete ParNameCTLocListPtr; ParNameCTLocListPtr = NULL; } - + + if ($8) + pNode->setHiveOptions(*$8); + $$ = pNode; - delete $1; /*TableTokens*/ - delete $2 /*ddl_qualified_name*/; + delete $1; //TableTokens + delete $2; //ddl_qualified_name } | create_table_start_tokens ddl_qualified_name + { + if (CmpCommon::getDefault(HIVE_CTAS_IN_NATIVE_MODE) == DF_OFF) + SqlParser_CurrentParser->hiveDDLInfo_->setFoundDDL(FALSE); + } optional_create_table_attribute_list create_table_as_attr_list_end ctas_load_and_in_memory_options ctas_insert_columns + optional_hive_options create_table_as_token optional_locking_stmt_list query_expression @@ -25292,16 +25272,7 @@ table_definition : create_table_start_tokens ddl_qualified_name YYERROR; } - if (($1->getType() == TableTokens::TYPE_EXTERNAL_HIVE_TABLE) || - ($1->getType() == TableTokens::TYPE_MANAGED_HIVE_TABLE)) - { - // create hive table .. as.. not supported. - *SqlParser_Diags << DgSqlCode(-3242) - << DgString0("'create hive table ... as ...' construct is not allowed."); - YYERROR; - } - - $1->setOptions($5); + $1->setOptions($6); if ($1->isVolatile()) qn = processVolatileDDLName($2, FALSE, FALSE); else @@ -25309,9 +25280,9 @@ table_definition : create_table_start_tokens ddl_qualified_name if (! qn) YYABORT; - RelRoot *top = finalize($9); + RelRoot *top = finalize($11); //limit clause - if ($10) + if ($12) { if (top->getFirstNRows() >= 0) { @@ -25321,16 +25292,16 @@ table_definition : create_table_start_tokens ddl_qualified_name else { NABoolean negate; - if ($10->castToConstValue(negate)) + if ($12->castToConstValue(negate)) { - ConstValue * limit = (ConstValue*)$10; + ConstValue * limit = (ConstValue*)$12; Lng32 scale = 0; top->setFirstNRows(limit->getExactNumericValue(scale)); top->setFirstNRowsParam(NULL); } else { - top->setFirstNRowsParam($10); + top->setFirstNRowsParam($12); top->setFirstNRows(-1); } } @@ -25339,11 +25310,10 @@ table_definition : create_table_start_tokens ddl_qualified_name StmtDDLCreateTable *pNode = new (PARSERHEAP()) StmtDDLCreateTable( - *qn /*ddl_qualified_name*/, - NULL /*table_definition_body*/, - $3 /*optional_create_table_ - *attribute_list*/, - $6, /* insert column list */ + *qn, //ddl_qualified_name + NULL, //table_definition_body + $4, //optional_create_table_attribute_list + $7, //insert column list top, PARSERHEAP()); $1->setTableTokens(pNode); @@ -25361,9 +25331,12 @@ table_definition : create_table_start_tokens ddl_qualified_name ParNameCTLocListPtr = NULL; } + if ($8) + pNode->setHiveOptions(*$8); + $$ = pNode; - delete $1; /*TableTokens*/ - delete $2 /*ddl_qualified_name*/; + delete $1; //TableTokens + delete $2; //ddl_qualified_name } | TOK_CREATE TOK_HBASE TOK_TABLE identifier '(' col_fam_quoted_string_list ')' @@ -25513,20 +25486,6 @@ create_table_start_tokens : $$ = tableTokens; } - | TOK_CREATE TOK_HIVE TOK_TABLE optional_if_not_exists_clause - { - ParNameCTLocListPtr = new (PARSERHEAP()) - ParNameLocList(SQLTEXT(), (CharInfo::CharSet)SQLTEXTCHARSET(), SQLTEXTW(), PARSERHEAP()); - TableTokens *tableTokens = new TableTokens(TableTokens::TYPE_MANAGED_HIVE_TABLE, $4); - $$ = tableTokens; - } - | TOK_CREATE TOK_EXTERNAL TOK_HIVE TOK_TABLE optional_if_not_exists_clause - { - ParNameCTLocListPtr = new (PARSERHEAP()) - ParNameLocList(SQLTEXT(), (CharInfo::CharSet)SQLTEXTCHARSET(), SQLTEXTW(), PARSERHEAP()); - TableTokens *tableTokens = new TableTokens(TableTokens::TYPE_EXTERNAL_HIVE_TABLE, $5); - $$ = tableTokens; - } /* type boolean */ optional_if_not_exists_clause : empty @@ -26672,11 +26631,6 @@ like_option : TOK_WITHOUT TOK_CONSTRAINTS $$ = new (PARSERHEAP()) ElemDDLLikeOptWithoutLobColumns(); } - | TOK_WITH TOK_HIVE TOK_OPTIONS QUOTED_STRING - { - $$ = new (PARSERHEAP()) - ElemDDLLikeOptWithHiveOptions(*$4); - } /* type pElemDDL */ optional_create_table_attribute_list : create_table_as_attr_list_start http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/regress/hive/DIFF008.KNOWN ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/DIFF008.KNOWN b/core/sql/regress/hive/DIFF008.KNOWN index fc40bdd..fc98a17 100644 --- a/core/sql/regress/hive/DIFF008.KNOWN +++ b/core/sql/regress/hive/DIFF008.KNOWN @@ -1,4 +1,4 @@ -283c283,286 +318c318,321 < --- SQL command prepared. --- > *** ERROR[4002] Column T00804.A is not found. Table T00804 not exposed. > Tables in scope: HIVE.HIVE.T00804. Default schema: HIVE.SCH008. http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/regress/hive/EXPECTED008 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED008 b/core/sql/regress/hive/EXPECTED008 index 32e9b0f..ce76529 100644 --- a/core/sql/regress/hive/EXPECTED008 +++ b/core/sql/regress/hive/EXPECTED008 @@ -97,7 +97,13 @@ A B >>-- Use regrhive until that issue is fixed. >>sh echo "create table sch008.t00803 as select * from sch008.t00802;" > >>TEST008_junk; >>sh regrhive.ksh -f TEST008_junk; ->>--create table t00803 as select * from sch008.t00802; +>>cqd hive_ctas_in_native_mode 'ON'; + +--- SQL operation complete. +>>-- create table t00803 as select * from sch008.t00802; +>>cqd hive_ctas_in_native_mode reset; + +--- SQL operation complete. >>invoke t00803; -- Definition of hive table HIVE.SCH008.T00803 @@ -119,6 +125,35 @@ A B --- 1 row(s) selected. >> +>>-- ctas create executed in Hive and insert executed in sql +>>drop table if exists t008032; + +--- SQL operation complete. +>>explain options 'f' create table t008032 no load as select * from t00802; + +LC RC OP OPERATOR OPT DESCRIPTION CARD +---- ---- ---- -------------------- -------- -------------------- --------- + +1 . 2 root 1.00E+000 +. . 1 create_table_as 1.00E+000 + +--- SQL operation complete. +>>create table t008032 no load as select * from sch008.t00802; + +--- 0 row(s) inserted. +>>invoke t008032; + +-- Definition of hive table HIVE.SCH008.T008032 +-- Definition current Fri Jun 15 05:38:24 2018 + + ( + A INT + , B INT + ) + /* stored as textfile */ + +--- SQL operation complete. +>> >>-- describe will show datatype 132 for b >>prepare s from insert into t00802 values (?, ?); @@ -289,6 +324,7 @@ Objects in Schema HIVE.SCH008 t00802 t00803 +t008032 v00802 v00803 @@ -311,6 +347,7 @@ Objects in Schema HIVE.SCH008 t00802 t00803 +t008032 v00803 --- SQL operation complete. @@ -401,18 +438,101 @@ REGISTER /*INTERNAL*/ HIVE TABLE HIVE.SCH008.T00805; --- SQL operation complete. >> ->>-- create external hive table LIKE a traf table +>>-- create external hive table like a traf table >>drop table if exists hive.sch008.columns; --- SQL operation complete. ->>create external hive table hive.sch008.columns like trafodion."_MD_".columns -+> with hive options 'stored as sequencefile'; - ---- SQL operation complete. ->>create external hive table if not exists hive.sch008.columns like trafodion."_MD_".columns -+> with hive options 'stored as sequencefile'; - ---- SQL operation complete. +>>explain create external table hive.sch008.columns no load ++> hive options 'stored as sequencefile' ++> as select * from trafodion."_MD_".columns; + +------------------------------------------------------------------ PLAN SUMMARY +MODULE_NAME .............. DYNAMICALLY COMPILED +STATEMENT_NAME ........... NOT NAMED +PLAN_ID .................. 212395845354479165 +ROWS_OUT ................. 1 +EST_TOTAL_COST ........... 0 +STATEMENT ................ create external table hive.sch008.columns no load + hive options 'stored as sequencefile' as + select * from trafodion."_MD_".columns; + + +------------------------------------------------------------------ NODE LISTING +ROOT ====================================== SEQ_NO 2 ONLY CHILD 1 +REQUESTS_IN .............. 1 +ROWS_OUT ................. 1 +EST_OPER_COST ............ 0 +EST_TOTAL_COST ........... 0 +DESCRIPTION + est_memory_per_node .... 10240.00(Limit), 0.00(BMOs), 0.00(nBMOs) MB + max_card_est ........... 1 + fragment_id ............ 0 + parent_frag ............ (none) + fragment_type .......... master + affinity_value ......... 0 + max_max_cardinality .... 1 + xn_access_mode ......... read_only + xn_autoabort_interval 0 + auto_query_retry ....... enabled for privilege checks + plan_version ....... 2,600 + embedded_arkcmp ........ used + IS_SQLCI ............... ON + LDAP_USERNAME + MODE_SEABASE ........... ON + SEABASE_VOLATILE_TABLES ON + HBASE_ASYNC_DROP_TABLE OFF + HBASE_SERIALIZATION .... ON + HBASE_SMALL_SCANNER .... SYSTEM + HBASE_FILTER_PREDS ..... 2 + TRAF_ALIGNED_ROW_FORMAT ON + TRAF_INDEX_CREATE_OPT ON + TRAF_DDL_ON_HIVE_OBJECT ON + SCHEMA ................. HIVE.SCH008 + GENERATE_EXPLAIN ....... ON + + +CREATE_TABLE_AS =========================== SEQ_NO 1 NO CHILDREN +REQUESTS_IN .............. 1 +ROWS_OUT ................. 1 +EST_OPER_COST ............ 0 +EST_TOTAL_COST ........... 0 +DESCRIPTION + max_card_est ........... -1 + fragment_id ............ 0 + parent_frag ............ (none) + fragment_type .......... master + CreateQuery ............ CREATE EXTERNAL TABLE HIVE.SCH008.COLUMNS ( + OBJECT_UID bigint, COLUMN_NAME varchar(256), + COLUMN_NUMBER int, COLUMN_CLASS char(2), + FS_DATA_TYPE int, SQL_DATA_TYPE char(32), + COLUMN_SIZE int, COLUMN_PRECISION int, + COLUMN_SCALE int, DATETIME_START_FIELD int, + DATETIME_END_FIELD int, IS_UPSHIFTED char(2), + COLUMN_FLAGS int, NULLABLE int, + CHARACTER_SET char(40), DEFAULT_CLASS int, + DEFAULT_VALUE varchar(1024), + COLUMN_HEADING varchar(256), + HBASE_COL_FAMILY varchar(40), + HBASE_COL_QUALIFIER varchar(40), + DIRECTION char(2), IS_OPTIONAL char(2), + FLAGS bigint ) stored as sequencefile + InsertQuery ............ insert into HIVE.SCH008.COLUMNS select * from + trafodion."_MD_".columns; + UpsertLoadQuery ........ NULL + UpdStatsQuery .......... UPDATE STATISTICS FOR TABLE HIVE.SCH008.COLUMNS ON + EVERY KEY SAMPLE SET ROWCOUNT %Ld; + +--- SQL operation complete. +>>create external table hive.sch008.columns no load ++> hive options 'stored as sequencefile' ++> as select * from trafodion."_MD_".columns; + +--- 0 row(s) inserted. +>>create external table if not exists hive.sch008.columns no load ++> hive options 'stored as sequencefile' ++> as select * from trafodion."_MD_".columns; + +--- 0 row(s) inserted. >>showddl hive.sch008.columns; /* Hive DDL */ @@ -451,14 +571,16 @@ REGISTER /*INTERNAL*/ HIVE TABLE HIVE.SCH008.COLUMNS; /* ObjectUID = 1402470925633639085 */ --- SQL operation complete. +>> >>-- create managed hive table LIKE traf table >>drop table if exists hive.sch008.columns; --- SQL operation complete. ->>create hive table hive.sch008.columns like trafodion."_MD_".columns -+> with hive options 'stored as sequencefile'; +>>create table hive.sch008.columns no load ++> hive options 'stored as sequencefile' ++> as select * from trafodion."_MD_".columns; ---- SQL operation complete. +--- 0 row(s) inserted. >>showddl hive.sch008.columns; /* Hive DDL */ @@ -498,6 +620,51 @@ REGISTER /*INTERNAL*/ HIVE TABLE HIVE.SCH008.COLUMNS; --- SQL operation complete. >> +>>-- ctas stmt is passed to Hive layer. +>>drop table if exists hive.hive.tempsrc; + +--- SQL operation complete. +>>drop table if exists hive.hive.temptgt; + +--- SQL operation complete. +>>create table hive.hive.tempsrc(a int); + +--- SQL operation complete. +>>cqd hive_ctas_in_native_mode 'ON'; + +--- SQL operation complete. +>>explain options 'f' create external table hive.hive.temptgt as select * from tempsrc; + +LC RC OP OPERATOR OPT DESCRIPTION CARD +---- ---- ---- -------------------- -------- -------------------- --------- + +1 . 2 root 1.00E+000 +. . 1 hive_ddl 1.00E+000 + +--- SQL operation complete. +>>create external table hive.hive.temptgt as select * from tempsrc; + +*** ERROR[1214] Error encountered when executing HiveQL statement "create external table `default`.TEMPTGT as select * from tempsrc". Details: org.apache.hive.service.cli.HiveSQLException: Error while compiling statement: FAILED: SemanticException [Error 10070]: CREATE-TABLE-AS-SELECT cannot create external table + +--- SQL operation failed with errors. +>>create table hive.hive.temptgt as select * from tempsrc; + +--- SQL operation complete. +>>invoke hive.hive.temptgt; + +-- Definition of hive table HIVE.HIVE.TEMPTGT +-- Definition current Fri Jun 15 18:12:41 2018 + + ( + A INT + ) + /* stored as textfile */ + +--- SQL operation complete. +>>cqd hive_ctas_in_native_mode reset; + +--- SQL operation complete. +>> >>-- create and drop hive table and external table >>drop table if exists t00806; @@ -792,9 +959,59 @@ t00807.a t00807.b >>sh regrhive.ksh -f TEST008_junk | tee -a LOG008; t00807.a t00807.b >> +>>explain options 'f' create table hive.sch008.t008072 as select * from hive.sch008.t00807; + +LC RC OP OPERATOR OPT DESCRIPTION CARD +---- ---- ---- -------------------- -------- -------------------- --------- + +1 . 2 root 1.00E+000 +. . 1 create_table_as 1.00E+000 + +--- SQL operation complete. +>>create table hive.sch008.t008072 as select * from hive.sch008.t00807; + +--- 0 row(s) inserted. +>> +>>drop table t008072; + +--- SQL operation complete. +>> +>>drop table if exists trafodion.sch.t00807; + +--- SQL operation complete. +>>create table trafodion.sch.t00807 (a int); + +--- SQL operation complete. +>>explain options 'f' create table hive.sch008.t008072 as select * from trafodion.sch.t00807; + +LC RC OP OPERATOR OPT DESCRIPTION CARD +---- ---- ---- -------------------- -------- -------------------- --------- + +1 . 2 root 1.00E+000 +. . 1 create_table_as 1.00E+000 + +--- SQL operation complete. +>>create table hive.sch008.t008072 as select * from trafodion.sch.t00807; + +--- 0 row(s) inserted. +>>invoke hive.sch008.t008072; + +-- Definition of hive table HIVE.SCH008.T008072 +-- Definition current Fri Jun 15 05:43:02 2018 + + ( + A INT + ) + /* stored as textfile */ + +--- SQL operation complete. +>> >>drop table t00807; --- SQL operation complete. +>>drop table t008072; + +--- SQL operation complete. >> >> >>-- explain on hive DDL @@ -1030,50 +1247,10 @@ alter database trafodion.temp; --- SQL operation failed with errors. >> ->>-- return error: ctas on hive cannot use traf in the SELECT query +>>-- return error: table temp does not exist >>create table hive.hive.temptab2 as select * from trafodion.sch.temp; -*** ERROR[1214] Error encountered when executing HiveQL statement "create table `default`.TEMPTAB2 as select * from trafodion.sch.temp". Details: org.apache.hive.service.cli.HiveSQLException: Error while compiling statement: FAILED: ParseException line 1:62 cannot recognize input near '.' 'temp' '<EOF>' in table source - ---- SQL operation failed with errors. ->> ->>-- cannot use 'with hive options' with non-hive tables ->>create table trafodion.seabase.temp like trafodion."_MD_".objects -+> with hive options 'abc'; - -*** ERROR[3242] This statement is not supported. Reason: Hive options cannot be specified for this table. - ---- SQL operation failed with errors. ->> ->>-- target table must be a hive table ->>create external hive table trafodion.seabase.temp like trafodion."_MD_".objects; - -*** ERROR[3242] This statement is not supported. Reason: LIKE target table must be a hive table. - -*** ERROR[8822] The statement was not prepared. - ->> ->>-- source table must be a trafodion table ->>create external hive table hive.hive.temp like hive.hive.temp2 -+> with hive options 'abc'; - -*** ERROR[3242] This statement is not supported. Reason: LIKE source table must be a trafodion table. - -*** ERROR[8822] The statement was not prepared. - ->> ->>-- ctas tgt cannot be an external hive table ->>create external hive table hive.hive.temp as select * from trafodion."_MD_".objects; - -*** ERROR[3242] This statement is not supported. Reason: 'create hive table ... as ...' construct is not allowed. - -*** ERROR[8822] The statement was not prepared. - ->> ->>-- like option must be specified ->>create external hive table hive.hive.temp (a int); - -*** ERROR[3242] This statement is not supported. Reason: LIKE clause must be specified to create this Hive table. +*** ERROR[4082] Object TRAFODION.SCH.TEMP does not exist or is inaccessible. *** ERROR[8822] The statement was not prepared. http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/regress/hive/FILTER008 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/FILTER008 b/core/sql/regress/hive/FILTER008 index 1dcb5b9..9dcb6c4 100755 --- a/core/sql/regress/hive/FILTER008 +++ b/core/sql/regress/hive/FILTER008 @@ -32,4 +32,5 @@ fi sed " s/\/\* ObjectUID = *[0-9]*/ObjectUID = <UID removed>/g +s/est_memory_per_node[ ]*[\.]* .*/est_memory_per_node... removed/g " $fil http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/regress/hive/TEST008 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST008 b/core/sql/regress/hive/TEST008 index 61dfd60..b84b1e2 100644 --- a/core/sql/regress/hive/TEST008 +++ b/core/sql/regress/hive/TEST008 @@ -50,10 +50,18 @@ select * from t00802; -- Use regrhive until that issue is fixed. sh echo "create table sch008.t00803 as select * from sch008.t00802;" > TEST008_junk; sh regrhive.ksh -f TEST008_junk; ---create table t00803 as select * from sch008.t00802; +cqd hive_ctas_in_native_mode 'ON'; +-- create table t00803 as select * from sch008.t00802; +cqd hive_ctas_in_native_mode reset; invoke t00803; select * from t00803; +-- ctas create executed in Hive and insert executed in sql +drop table if exists t008032; +explain options 'f' create table t008032 no load as select * from t00802; +create table t008032 no load as select * from sch008.t00802; +invoke t008032; + -- describe will show datatype 132 for b prepare s from insert into t00802 values (?, ?); describe s; @@ -108,19 +116,37 @@ drop table if exists hive.sch008.`t00805like2`; create table hive."sch008"."t00805" (`a` int); showddl hive.sch008.t00805; --- create external hive table LIKE a traf table +-- create external hive table like a traf table drop table if exists hive.sch008.columns; -create external hive table hive.sch008.columns like trafodion."_MD_".columns - with hive options 'stored as sequencefile'; -create external hive table if not exists hive.sch008.columns like trafodion."_MD_".columns - with hive options 'stored as sequencefile'; +explain create external table hive.sch008.columns no load + hive options 'stored as sequencefile' + as select * from trafodion."_MD_".columns; +create external table hive.sch008.columns no load + hive options 'stored as sequencefile' + as select * from trafodion."_MD_".columns; +create external table if not exists hive.sch008.columns no load + hive options 'stored as sequencefile' + as select * from trafodion."_MD_".columns; showddl hive.sch008.columns; + -- create managed hive table LIKE traf table drop table if exists hive.sch008.columns; -create hive table hive.sch008.columns like trafodion."_MD_".columns - with hive options 'stored as sequencefile'; +create table hive.sch008.columns no load + hive options 'stored as sequencefile' + as select * from trafodion."_MD_".columns; showddl hive.sch008.columns; +-- ctas stmt is passed to Hive layer. +drop table if exists hive.hive.tempsrc; +drop table if exists hive.hive.temptgt; +create table hive.hive.tempsrc(a int); +cqd hive_ctas_in_native_mode 'ON'; +explain options 'f' create external table hive.hive.temptgt as select * from tempsrc; +create external table hive.hive.temptgt as select * from tempsrc; +create table hive.hive.temptgt as select * from tempsrc; +invoke hive.hive.temptgt; +cqd hive_ctas_in_native_mode reset; + -- create and drop hive table and external table drop table if exists t00806; create table t00806 (a int, b string); @@ -194,7 +220,19 @@ truncate table t00807; sh echo "select * from sch008.t00807;" > TEST008_junk; sh regrhive.ksh -f TEST008_junk | tee -a LOG008; +explain options 'f' create table hive.sch008.t008072 as select * from hive.sch008.t00807; +create table hive.sch008.t008072 as select * from hive.sch008.t00807; + +drop table t008072; + +drop table if exists trafodion.sch.t00807; +create table trafodion.sch.t00807 (a int); +explain options 'f' create table hive.sch008.t008072 as select * from trafodion.sch.t00807; +create table hive.sch008.t008072 as select * from trafodion.sch.t00807; +invoke hive.sch008.t008072; + drop table t00807; +drop table t008072; -- explain on hive DDL @@ -270,26 +308,9 @@ alter database trafodion.temp; -- return error: unsupported operation from 'process hive ddl' stmt process hive ddl 'grnt abc'; --- return error: ctas on hive cannot use traf in the SELECT query +-- return error: table temp does not exist create table hive.hive.temptab2 as select * from trafodion.sch.temp; --- cannot use 'with hive options' with non-hive tables -create table trafodion.seabase.temp like trafodion."_MD_".objects - with hive options 'abc'; - --- target table must be a hive table -create external hive table trafodion.seabase.temp like trafodion."_MD_".objects; - --- source table must be a trafodion table -create external hive table hive.hive.temp like hive.hive.temp2 - with hive options 'abc'; - --- ctas tgt cannot be an external hive table -create external hive table hive.hive.temp as select * from trafodion."_MD_".objects; - --- like option must be specified -create external hive table hive.hive.temp (a int); - drop database hive.sch008 cascade; drop schema if exists hive.sch008 cascade; showddl schema hive.sch008; http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/sqlcomp/CmpDescribe.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpDescribe.cpp b/core/sql/sqlcomp/CmpDescribe.cpp index 26e0deb..61faa54 100644 --- a/core/sql/sqlcomp/CmpDescribe.cpp +++ b/core/sql/sqlcomp/CmpDescribe.cpp @@ -2516,98 +2516,6 @@ short CmpDescribeHiveTable ( return 0; } -// this method is used to convert a trafodion table definition to corresponding -// Hive 'create' DDL. -// Used when a Hive table is being create 'like' a traf table. -short CmpDescribeTrafAsHiveTable ( - const CorrName &dtName, - short type, - char* &outbuf, - ULng32 &outbuflen, - CollHeap *heap, - UInt32 columnLengthLimit) -{ - BindWA bindWA(ActiveSchemaDB(), CmpCommon::context(), FALSE/*inDDL*/); - NATable *naTable = bindWA.getNATable((CorrName&)dtName); - if (naTable == NULL || bindWA.errStatus()) - return -1; - - if (NOT naTable->isSeabaseTable()) - return -1; - - char * buf = new (heap) char[15000]; - CMPASSERT(buf); - - time_t tp; - time(&tp); - - Space space; - - if (!CmpDescribeIsAuthorized(SQLOperation::UNKNOWN, - naTable->getPrivInfo(), - COM_BASE_TABLE_OBJECT)) - return -1; - - // emit an initial newline - outputShortLine(space, " "); - - outputShortLine(space, " ( "); - - Int32 ii = 0; - for (Int32 i = 0; i < (Int32)naTable->getColumnCount(); i++) - { - NAColumn * nac = naTable->getNAColumnArray()[i]; - - if (nac->isSystemColumn()) - continue; - - NAString colName = nac->getColName(); - colName.toLower(); - - const NAType * nat = nac->getType(); - - sprintf(buf, "%s ", colName.data()); - - NAString nas; - nat->getMyTypeAsHiveText(&nas); - - // if it is a character type and it is longer than the length - // limit in bytes, then shorten the target type - if ((nat->getTypeQualifier() == NA_CHARACTER_TYPE) && - (!nat->isLob()) && - (columnLengthLimit < UINT_MAX)) - { - const CharType * natc = (const CharType *)nat; - if (natc->getDataStorageSize() > columnLengthLimit) - { - CharType * newType = (CharType *)natc->newCopy(NULL); - newType->setDataStorageSize(columnLengthLimit); - nas.clear(); - newType->getMyTypeAsText(&nas, FALSE); - delete newType; - } - } - - sprintf(&buf[strlen(buf)], "%s", nas.data()); - - NAString colString(buf); - Int32 j = ii; - outputColumnLine(space, colString, j); - - ii++; - } - - outputShortLine(space, " )"); - - outbuflen = space.getAllocatedSpaceSize(); - outbuf = new (heap) char[outbuflen]; - space.makeContiguous(outbuf, outbuflen); - - NADELETEBASIC(buf, heap); - - return 0; -} - // type: 1, invoke. 2, showddl. 3, create_like short cmpDisplayColumn(const NAColumn *nac, char * inColName, http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/sqlcomp/CmpDescribe.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpDescribe.h b/core/sql/sqlcomp/CmpDescribe.h index 532001f..0903759 100644 --- a/core/sql/sqlcomp/CmpDescribe.h +++ b/core/sql/sqlcomp/CmpDescribe.h @@ -91,14 +91,6 @@ short CmpDescribeHiveTable ( NABoolean isDetail = FALSE, UInt32 columnLengthLimit = UINT_MAX); -short CmpDescribeTrafAsHiveTable ( - const CorrName &dtName, - short type, // 1, invoke. 2, showddl. 3, createLike - char* &outbuf, - ULng32 &outbuflen, - CollHeap *heap, - UInt32 columnLengthLimit = UINT_MAX); - // type: 1, invoke. 2, showddl. 3, create_like extern short cmpDisplayColumn(const NAColumn *nac, char * inColName, http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/sqlcomp/CmpSeabaseDDL.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpSeabaseDDL.h b/core/sql/sqlcomp/CmpSeabaseDDL.h index 346b6dc..785e6a0 100644 --- a/core/sql/sqlcomp/CmpSeabaseDDL.h +++ b/core/sql/sqlcomp/CmpSeabaseDDL.h @@ -1086,11 +1086,6 @@ protected: const ComObjectName &tgtTableName, const ComObjectName &srcTableName); - static short genDDLforHiveTableLikeTrafTable( - StmtDDLCreateTable * createTableNode, - NAString &currCatName, NAString &currSchName, - NAString &tableDDL); // output. Contains hive DDL string. - public: static NABoolean setupQueryTreeForHiveDDL( Parser::HiveDDLInfo * hiveDDLInfo, http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp index 3e9f1a2..3e9082d 100644 --- a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp +++ b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp @@ -11487,7 +11487,6 @@ void CmpSeabaseDDL::processDDLonHiveObjects(StmtDDLonHiveObjects * hddl, // Start error checks if (NOT ((hddl->getOper() == StmtDDLonHiveObjects::CREATE_) || - (hddl->getOper() == StmtDDLonHiveObjects::CREATE_LIKE_TRAF_) || (hddl->getOper() == StmtDDLonHiveObjects::DROP_) || (hddl->getOper() == StmtDDLonHiveObjects::ALTER_) || (hddl->getOper() == StmtDDLonHiveObjects::TRUNCATE_))) @@ -11617,8 +11616,7 @@ void CmpSeabaseDDL::processDDLonHiveObjects(StmtDDLonHiveObjects * hddl, } } - if ((hddl->getOper() == StmtDDLonHiveObjects::CREATE_) || - (hddl->getOper() == StmtDDLonHiveObjects::CREATE_LIKE_TRAF_)) + if (hddl->getOper() == StmtDDLonHiveObjects::CREATE_) { if (objExists) { @@ -11692,7 +11690,6 @@ void CmpSeabaseDDL::processDDLonHiveObjects(StmtDDLonHiveObjects * hddl, if ((CmpCommon::getDefault(HIVE_NO_REGISTER_OBJECTS) == DF_OFF) && ((hddl->getOper() == StmtDDLonHiveObjects::CREATE_) || - (hddl->getOper() == StmtDDLonHiveObjects::CREATE_LIKE_TRAF_) || (hddl->getOper() == StmtDDLonHiveObjects::ALTER_)) && (NOT isRegistered)) { @@ -11770,102 +11767,6 @@ label_error: return; } -short CmpSeabaseDDL::genDDLforHiveTableLikeTrafTable(StmtDDLCreateTable * createTableNode, - NAString &currCatName, NAString &currSchName, - NAString &tableDDL) -{ - Lng32 cliRC = 0; - Lng32 retcode = 0; - - ComObjectName tgtTableName(createTableNode->getTableName(), COM_TABLE_NAME); - ComAnsiNamePart currCatAnsiName(currCatName); - ComAnsiNamePart currSchAnsiName(currSchName); - tgtTableName.applyDefaults(currCatAnsiName, currSchAnsiName); - - if (tgtTableName.getCatalogNamePartAsAnsiString() != HIVE_SYSTEM_CATALOG) - { - *CmpCommon::diags() - << DgSqlCode(-3242) - << DgString0("LIKE target table must be a hive table."); - return -1; - } - - NAString tgtSchName; - - if ((tgtTableName.getSchemaNamePartAsAnsiString(TRUE).compareTo(HIVE_DEFAULT_SCHEMA_EXE, NAString::ignoreCase) == 0) || - (tgtTableName.getSchemaNamePartAsAnsiString(TRUE).compareTo(HIVE_SYSTEM_SCHEMA, NAString::ignoreCase) == 0)) - tgtSchName = HIVE_SYSTEM_SCHEMA; - else - tgtSchName = tgtTableName.getSchemaNamePartAsAnsiString(TRUE); - - ComObjectName srcTableName(createTableNode->getLikeSourceTableName(), COM_TABLE_NAME); - - srcTableName.applyDefaults(currCatName, currSchName); - - const NAString srcCatNamePart = srcTableName.getCatalogNamePartAsAnsiString(); - const NAString srcSchNamePart = srcTableName.getSchemaNamePartAsAnsiString(TRUE); - const NAString srcObjNamePart = srcTableName.getObjectNamePartAsAnsiString(TRUE); - CorrName srcCN(srcObjNamePart, STMTHEAP, srcSchNamePart, srcCatNamePart); - - if (NOT createTableNode->getIsLikeOptionSpecified()) - { - *CmpCommon::diags() - << DgSqlCode(-3242) - << DgString0("LIKE clause must be specified to create this hive table."); - return -1; - } - - ParDDLLikeOptsCreateTable &likeOptions = createTableNode->getLikeOptions(); - - if (srcTableName.getCatalogNamePartAsAnsiString() != TRAFODION_SYSCAT_LIT) - { - *CmpCommon::diags() - << DgSqlCode(-3242) - << DgString0("LIKE source table must be a trafodion table."); - return -1; - } - - if (createTableNode->managedHiveTable()) - tableDDL = "CREATE TABLE "; - else - tableDDL = "CREATE EXTERNAL TABLE "; - if (createTableNode->createIfNotExists()) - tableDDL += " IF NOT EXISTS "; - - tableDDL += (tgtSchName == HIVE_SYSTEM_SCHEMA ? "`default`" : tgtSchName) + "."; - tableDDL += tgtTableName.getObjectNamePartAsAnsiString(TRUE); - - char * buf = NULL; - ULng32 buflen = 0; - retcode = CmpDescribeTrafAsHiveTable(srcCN, 3/*createlike*/, - buf, buflen, - STMTHEAP); - if (retcode) - return -1; - - NABoolean done = FALSE; - Lng32 curPos = 0; - while (NOT done) - { - short len = *(short*)&buf[curPos]; - NAString frag(&buf[curPos+sizeof(short)], - len - ((buf[curPos+len-1]== '\n') ? 1 : 0)); - - tableDDL += frag; - curPos += ((((len+sizeof(short))-1)/8)+1)*8; - - if (curPos >= buflen) - done = TRUE; - } - - if (NOT likeOptions.getLikeOptHiveOptions().isNull()) - { - tableDDL += " " + likeOptions.getLikeOptHiveOptions(); - } - - return 0; -} - // ------------------------------------------------------------------------ // setupQueryTreeForHiveDDL // @@ -11903,43 +11804,6 @@ NABoolean CmpSeabaseDDL::setupQueryTreeForHiveDDL( StmtDDLonHiveObjects::ObjectType type = (StmtDDLonHiveObjects::ObjectType)hiveDDLInfo->ddlObjectType_; - if ((oper == StmtDDLonHiveObjects::CREATE_LIKE_TRAF_) && - (NOT ((hiveDDLInfo->essd_ == Parser::HiveDDLInfo::EXPLAIN_) || - (hiveDDLInfo->essd_ == Parser::HiveDDLInfo::SHOWPLAN_) || - (hiveDDLInfo->essd_ == Parser::HiveDDLInfo::SHOWSHAPE_)))) - { - ExprNode *hlt = *node; - if (hlt == NULL) - return FALSE; // node must be passed in. - - StmtQuery * stmt = (StmtQuery*)hlt->castToStatementExpr(); - RelRoot * root = (RelRoot*)stmt->getQueryExpression(); - DDLExpr * ddl = (DDLExpr*)root->child(0)->castToRelExpr(); - StmtDDLonHiveObjects * doh = - ddl->getDDLNode()->castToStmtDDLNode()->castToStmtDDLonHiveObjects(); - StmtDDLCreateTable * ct = doh->getChild(0)->castToStmtDDLNode()->castToStmtDDLCreateTable(); - if (! ct) - return FALSE; - - NAString tableDDL; - if (CmpSeabaseDDL::genDDLforHiveTableLikeTrafTable(ct, currCatName, currSchName, - tableDDL)) - return FALSE; - - doh->setHiveDDL(tableDDL); - - // indicate that this is the root for the entire query - if (root) - { - if (hiveDDLInfo->essd_ == Parser::HiveDDLInfo::DISPLAY_) - ((RelRoot*)root)->setDisplayTree(TRUE); - } - - *node = stmt; - - return TRUE; - } - // position and length of the object name specified in the query. Lng32 hiveNamePos = hiveDDLInfo->ddlNamePos_; Lng32 hiveNameLen = hiveDDLInfo->ddlNameLen_; @@ -12026,7 +11890,6 @@ NABoolean CmpSeabaseDDL::setupQueryTreeForHiveDDL( // explain/showplan/showshape/display. // // Regular DDL query: - // HiveLikeTraf DDL query: // StmtQuery => RelRoot => DDLExpr => StmtDDLonHiveObjects // // explain query: http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/sqlcomp/DefaultConstants.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h index 1488f77..748ace0 100644 --- a/core/sql/sqlcomp/DefaultConstants.h +++ b/core/sql/sqlcomp/DefaultConstants.h @@ -3336,14 +3336,19 @@ enum DefaultConstants // if set, ddl from Traf interface on Hive objects is supported. TRAF_DDL_ON_HIVE_OBJECTS, - - // This enum constant must be the LAST one in the list; it's a count, - // not an Attribute (it's not IN DefaultDefaults; it's the SIZE of it)! + + // If set to TRUE, CTAS on Hive object(Create and Insert...select) is processed in Hive. + // If set to FALSE, Create is done in Hive, and Insert...select is done in Traf. + // Default is OFF. + HIVE_CTAS_IN_NATIVE_MODE, + // Size of byte[] in java when direct byteBuffer can't be used // Used to read compressed hdfs text files and to write // both compressed and uncompressed hdfs files HDFS_IO_INTERIM_BYTEARRAY_SIZE_IN_KB, + // This enum constant must be the LAST one in the list; it's a count, + // not an Attribute (it's not IN DefaultDefaults; it's the SIZE of it)! __NUM_DEFAULT_ATTRIBUTES }; http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/sqlcomp/nadefaults.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp index 7650726..a0f245c 100644 --- a/core/sql/sqlcomp/nadefaults.cpp +++ b/core/sql/sqlcomp/nadefaults.cpp @@ -1636,6 +1636,8 @@ SDDkwd__(EXE_DIAGNOSTIC_EVENTS, "OFF"), DD_____(HIVE_CATALOG, ""), + DDkwd__(HIVE_CTAS_IN_NATIVE_MODE, "OFF"), + DDkwd__(HIVE_DATA_MOD_CHECK, "ON"), DDkwd__(HIVE_DEFAULT_CHARSET, (char *)SQLCHARSETSTRING_UTF8), http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/sqlcomp/parser.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/parser.cpp b/core/sql/sqlcomp/parser.cpp index 20937bd..8ade55b 100644 --- a/core/sql/sqlcomp/parser.cpp +++ b/core/sql/sqlcomp/parser.cpp @@ -896,20 +896,10 @@ Int32 Parser::parseSQL // if a hive ddl object was found during parsing, generate ddl expr tree. // foundDDL_ could be set during successful parsing as well as for // a query which gave a syntax error. - if (SqlParser_CurrentParser->hiveDDLInfo_->ddlOperation_ == - StmtDDLonHiveObjects::CREATE_LIKE_TRAF_) - { - // TheParseTree is already set up with the needed structures. - // Any additional info would be added in processHiveDDL method. - *node = TheParseTree; - } - else - { - if (TheParseTree) - delete TheParseTree; - TheParseTree = NULL; - *node = NULL; - } + if (TheParseTree) + delete TheParseTree; + TheParseTree = NULL; + *node = NULL; if ((processHiveDDL(SqlParser_CurrentParser->hiveDDLInfo_, node)) && (*node != NULL)) http://git-wip-us.apache.org/repos/asf/trafodion/blob/bb3dee8f/core/sql/sqlcomp/parser.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/parser.h b/core/sql/sqlcomp/parser.h index bbb6da9..6189c20 100644 --- a/core/sql/sqlcomp/parser.h +++ b/core/sql/sqlcomp/parser.h @@ -323,6 +323,11 @@ ItemExpr *get_w_ItemExprTree(const NAWchar * str, ifExistsOrNotExists_ = ifExistsOrNotExists; } + void setFoundDDL(NABoolean v) + { + foundDDL_ = v; + } + // in some cases, parser should not do hive ddl check. // This may happen for internal parsing, for ex, for view expansion, // or internal MD ddl compiles.
