http://git-wip-us.apache.org/repos/asf/trafodion/blob/2bba19ff/core/sql/optimizer/RelExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExeUtil.cpp b/core/sql/optimizer/RelExeUtil.cpp index 971a537..2675470 100644 --- a/core/sql/optimizer/RelExeUtil.cpp +++ b/core/sql/optimizer/RelExeUtil.cpp @@ -395,10 +395,6 @@ const NAString ExeUtilExpr::getText() const result = "CREATE_TABLE_AS"; break; - case FAST_DELETE_: - result = "FAST_DELETE"; - break; - case HIVE_TRUNCATE_: result = "HIVE_TRUNCATE"; break; @@ -850,30 +846,25 @@ RelExpr * ExeUtilCreateTableAs::copyTopNode(RelExpr *derivedNode, } // ----------------------------------------------------------------------- -// Member functions for class ExeUtilFastDelete +// Member functions for class ExeUtilHiveTruncateLegacy // ----------------------------------------------------------------------- -RelExpr * ExeUtilFastDelete::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap) +RelExpr * ExeUtilHiveTruncateLegacy::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap) { - ExeUtilFastDelete *result; + ExeUtilHiveTruncateLegacy *result; if (derivedNode == NULL) - result = new (outHeap) ExeUtilFastDelete(getTableName(), - getExprNode(), NULL, CharInfo::UnknownCharSet, - doPurgedataCat_, - noLog_, - ignoreTrigger_, - isPurgedata_, - outHeap); + result = new (outHeap) ExeUtilHiveTruncateLegacy(getTableName(), + pl_, + outHeap); else - result = (ExeUtilFastDelete *) derivedNode; - - result->doParallelDelete_ = doParallelDelete_; - result->doParallelDeleteIfXn_ = doParallelDeleteIfXn_; - result->offlineTable_ = offlineTable_; - result->doLabelPurgedata_ = doLabelPurgedata_; + result = (ExeUtilHiveTruncateLegacy *) derivedNode; - result->numLOBs_ = numLOBs_; - result->lobNumArray_ = lobNumArray_; + result->hiveTableLocation_= hiveTableLocation_; + result->hiveHostName_ = hiveHostName_; + result->hiveHdfsPort_ = hiveHdfsPort_; + result->suppressModCheck_ = suppressModCheck_; + result->dropTableOnDealloc_ = dropTableOnDealloc_; + result->noSecurityCheck_ = noSecurityCheck_; return ExeUtilExpr::copyTopNode(result, outHeap); } @@ -887,21 +878,18 @@ RelExpr * ExeUtilHiveTruncate::copyTopNode(RelExpr *derivedNode, CollHeap* outHe if (derivedNode == NULL) result = new (outHeap) ExeUtilHiveTruncate(getTableName(), - pl_, + hiveTableName_, + hiveTruncQuery_, outHeap); else result = (ExeUtilHiveTruncate *) derivedNode; - result->hiveTableLocation_= hiveTableLocation_; - result->hiveHostName_ = hiveHostName_; - result->hiveHdfsPort_ = hiveHdfsPort_; - result->suppressModCheck_ = suppressModCheck_; result->dropTableOnDealloc_ = dropTableOnDealloc_; + result->noSecurityCheck_ = noSecurityCheck_; return ExeUtilExpr::copyTopNode(result, outHeap); } - // ----------------------------------------------------------------------- RelExpr * ExeUtilHiveQuery::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap) @@ -5263,60 +5251,9 @@ RelExpr * ExeUtilCreateTableAs::bindNode(BindWA *bindWA) return boundExpr; } // ----------------------------------------------------------------------- -// member functions for class ExeUtilFastDelete -// ----------------------------------------------------------------------- -RelExpr * ExeUtilFastDelete::bindNode(BindWA *bindWA) -{ - if (nodeIsBound()) - { - bindWA->getCurrentScope()->setRETDesc(getRETDesc()); - return this; - } - - bindChildren(bindWA); - if (bindWA->errStatus()) - return this; - - // do not do override schema for this - bindWA->setToOverrideSchema(FALSE); - - NATable * naTable = bindWA->getNATable(getTableName()); - if ((!naTable) || - (bindWA->errStatus())) - return this; - - if ((getTableName().isHive()) || - (naTable->isHiveTable())) - { - *CmpCommon::diags() << DgSqlCode(-3242) - << DgString0("Purgedata is not allowed for Hive tables. Use 'Truncate Table' command."); - bindWA->setErrStatus(); - return NULL; - } - - if (! getTableName().isSeabase()) - { - *CmpCommon::diags() << DgSqlCode(-4222) << DgString0("PURGEDATA"); - bindWA->setErrStatus(); - return NULL; - } - - DDLExpr * ddlExpr = new(bindWA->wHeap()) DDLExpr(NULL, - getStmtText(), - CharInfo::UnknownCharSet, - CmpCommon::statementHeap()); - ddlExpr->setPurgedata(TRUE); - ddlExpr->setPurgedataTableName(getTableName()); - - RelExpr * boundExpr = ddlExpr->bindNode(bindWA); - - return boundExpr; -} - -// ----------------------------------------------------------------------- -// member functions for class ExeUtilHiveTruncate +// member functions for class ExeUtilHiveTruncateLegacy // ----------------------------------------------------------------------- -RelExpr * ExeUtilHiveTruncate::bindNode(BindWA *bindWA) +RelExpr * ExeUtilHiveTruncateLegacy::bindNode(BindWA *bindWA) { if (nodeIsBound()) { @@ -5410,6 +5347,115 @@ RelExpr * ExeUtilHiveTruncate::bindNode(BindWA *bindWA) } // ----------------------------------------------------------------------- +// member functions for class ExeUtilHiveTruncate +// ----------------------------------------------------------------------- +RelExpr * ExeUtilHiveTruncate::bindNode(BindWA *bindWA) +{ + if (nodeIsBound()) + { + bindWA->getCurrentScope()->setRETDesc(getRETDesc()); + return this; + } + + bindChildren(bindWA); + if (bindWA->errStatus()) + return this; + + NATable *naTable = NULL; + + // do not do override schema for this + bindWA->setToOverrideSchema(FALSE); + + naTable = bindWA->getNATable(getTableName()); + if (getIfExist() && (! naTable)) + { + setTableNotExist(TRUE); + + bindWA->resetErrStatus(); + CmpCommon::diags()->clear(); + RelExpr * boundExpr = ExeUtilExpr::bindNode(bindWA); + if (bindWA->errStatus()) + return NULL; + + return boundExpr; + } + + if ((!naTable) || + (bindWA->errStatus())) + return this; + + if ((NOT getTableName().isHive()) || + (!naTable->isHiveTable())) + { + *CmpCommon::diags() << DgSqlCode(-3242) + << DgString0("Truncate is only allowed for hive tables."); + bindWA->setErrStatus(); + return NULL; + } + + // if no security check is to be done, skip it. + if (NOT noSecurityCheck_) + { + // In Hive, you need admin privs to truncate files. At this time, we don't + // know if the current user has admin privileges, return an error. + char * sentryEnv = getenv("SENTRY_SECURITY_FOR_HIVE"); + if (sentryEnv && strcmp(sentryEnv, "TRUE") == 0) + { + *CmpCommon::diags() << DgSqlCode(-3242) + << DgString0("Truncate must be performed through native Hive interface."); + bindWA->setErrStatus(); + return NULL; + } + + // If the current user has been granted the Trafodion Hive/DB root role or + // is DB__ROOT, allow the operation. + // If the current user has select and delete privileges, allow the operation + if (bindWA->currentCmpContext()->isAuthorizationEnabled()) + { + NABoolean found = FALSE; + if (ComUser::isRootUserID() || + ComUser::currentUserHasRole(HIVE_ROLE_ID) || + ComUser::currentUserHasRole(ROOT_ROLE_ID)) + found = TRUE; + + if (!found) + { + PrivMgrUserPrivs *pPrivInfo = naTable->getPrivInfo(); + if (pPrivInfo && + pPrivInfo->hasPriv(SELECT_PRIV) && + pPrivInfo->hasPriv(DELETE_PRIV)) + found = TRUE; + + if (!found) + { + *CmpCommon::diags() + << DgSqlCode( -1051 ) + << DgTableName(naTable->getTableName().getQualifiedNameAsAnsiString()); + bindWA->setErrStatus(); + return NULL; + } + } + } + } + + setHiveExternalTable(naTable->isHiveExternalTable()); + + // Allocate a TableDesc and attach it to this. + // +#ifdef __ignore + setUtilTableDesc(bindWA->createTableDesc(naTable, getTableName())); + if (bindWA->errStatus()) + return this; +#endif + + RelExpr * boundExpr = ExeUtilExpr::bindNode(bindWA); + if (bindWA->errStatus()) + return NULL; + + return boundExpr; +} + +// ----------------------------------------------------------------------- // member functions for class ExeUtilMaintainObject // ----------------------------------------------------------------------- RelExpr * ExeUtilMaintainObject::bindNode(BindWA *bindWA)
http://git-wip-us.apache.org/repos/asf/trafodion/blob/2bba19ff/core/sql/optimizer/RelExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExeUtil.h b/core/sql/optimizer/RelExeUtil.h index d94ab63..29588e2 100644 --- a/core/sql/optimizer/RelExeUtil.h +++ b/core/sql/optimizer/RelExeUtil.h @@ -371,6 +371,10 @@ public: {(v ? flags_ |= PURGEDATA : flags_ &= ~PURGEDATA); } NABoolean purgedata() { return (flags_ & PURGEDATA) != 0;} + void setPurgedataIfExists(NABoolean v) + {(v ? flags_ |= PURGEDATA_IF_EXISTS : flags_ &= ~PURGEDATA_IF_EXISTS); } + NABoolean purgedataIfExists() { return (flags_ & PURGEDATA_IF_EXISTS) != 0;} + // this ddlexpr is created for 'showddl <obj>, explain' to // explain the object explObjName. void setShowddlExplain(NABoolean v) @@ -411,6 +415,7 @@ public: SHOWDDL_EXPLAIN = 0x040000, SHOWDDL_EXPLAIN_INT = 0x080000, NO_LABEL_STATS = 0x100000, + PURGEDATA_IF_EXISTS = 0x200000, }; // see method processSpecialDDL in sqlcomp/parser.cpp @@ -486,7 +491,6 @@ public: CLEANUP_VOLATILE_TABLES_ = 5, GET_VOLATILE_INFO_ = 6, CREATE_TABLE_AS_ = 7, - FAST_DELETE_ = 8, GET_STATISTICS_ = 9, LONG_RUNNING_ = 11, GET_METADATA_INFO_ = 12, @@ -513,9 +517,10 @@ public: HBASE_UNLOAD_TASK_ = 36, ORC_FAST_AGGR_ = 37, GET_QID_ = 38, - HIVE_TRUNCATE_ = 39, + HIVE_TRUNCATE_LEGACY_ = 39, LOB_UPDATE_UTIL_ = 40, - HIVE_QUERY_ = 41 + HIVE_QUERY_ = 41, + HIVE_TRUNCATE_ = 45 }; ExeUtilExpr(ExeUtilType type, @@ -1017,36 +1022,23 @@ private: NABoolean deleteData_; }; -class ExeUtilFastDelete : public ExeUtilExpr +class ExeUtilHiveTruncateLegacy : public ExeUtilExpr { public: - ExeUtilFastDelete(const CorrName &name, - ExprNode * exprNode, - char * stmtText, - CharInfo::CharSet stmtTextCharSet, - NABoolean doPurgedataCat = FALSE, - NABoolean noLog = FALSE, - NABoolean ignoreTrigger = FALSE, - NABoolean isPurgedata = FALSE, - CollHeap *oHeap = CmpCommon::statementHeap()) - : ExeUtilExpr(FAST_DELETE_, name, exprNode, NULL, stmtText, stmtTextCharSet, oHeap), - doPurgedataCat_(doPurgedataCat), - noLog_(noLog), ignoreTrigger_(ignoreTrigger), - isPurgedata_(isPurgedata), - doParallelDelete_(FALSE), - doParallelDeleteIfXn_(FALSE), - offlineTable_(FALSE), - doLabelPurgedata_(FALSE), - numLOBs_(0), - lobNumArray_(oHeap) - { - }; - + ExeUtilHiveTruncateLegacy(const CorrName &name, + ConstStringList * pl, + CollHeap *oHeap = CmpCommon::statementHeap()) + : ExeUtilExpr(HIVE_TRUNCATE_LEGACY_, name, NULL, NULL, NULL, + CharInfo::UnknownCharSet, oHeap), + pl_(pl), suppressModCheck_(FALSE), dropTableOnDealloc_(FALSE), + noSecurityCheck_(FALSE) + { } + virtual NABoolean isExeUtilQueryType() { return TRUE; } - + virtual RelExpr * copyTopNode(RelExpr *derivedNode = NULL, CollHeap* outHeap = 0); - + virtual RelExpr * bindNode(BindWA *bindWAPtr); virtual RelExpr * preCodeGen(Generator * generator, @@ -1058,42 +1050,68 @@ public: virtual NABoolean aqrSupported() { return TRUE; } -private: - NABoolean doPurgedataCat_; + const NAString &getHiveTableLocation() const + { + return hiveTableLocation_; + } - NABoolean noLog_; - NABoolean ignoreTrigger_; + const NAString &getHiveHostName() const + { + return hiveHostName_; + } - NABoolean isPurgedata_; + const Int32 getHiveHdfsPort() const + { + return hiveHdfsPort_; + } - // do regular parallel delete at runtime. Start a Xn, if oen doesn't - // exist. - NABoolean doParallelDelete_; + ConstStringList* &partnList() { return pl_; } + + NABoolean getSuppressModCheck() const { return suppressModCheck_; } + NABoolean getDropTableOnDealloc() const { return dropTableOnDealloc_; } + NABoolean getNoSecurityCheck() const { return noSecurityCheck_; } - // do regular parallel delete if doParallelDelete is not chosen and - // there is a transaction running at runtime. - // If this is FALSE, then regular purgedata is invoked. - NABoolean doParallelDeleteIfXn_; + void setSuppressModCheck(NABoolean v=TRUE) { suppressModCheck_ = v; } + void setDropTableOnDealloc(NABoolean v=TRUE) { dropTableOnDealloc_ = v; } + void setNoSecurityCheck(NABoolean v) { noSecurityCheck_ = v; } +private: + NAString hiveTableLocation_; + NAString hiveHostName_; + Int32 hiveHdfsPort_; - NABoolean offlineTable_; + // timestamp of hiveTableLocation. + Int64 hiveModTS_; - // use the new parallel label purgedata operation. - NABoolean doLabelPurgedata_; - - // if there are LOB columns. - Lng32 numLOBs_; // number of LOB columns - NAList<short> lobNumArray_; // array of shorts. Each short is the lob num + // list of partitions to be truncated + ConstStringList * pl_; + NABoolean suppressModCheck_; + NABoolean dropTableOnDealloc_; + + // if this truncate node is added internally to process 'insert overwrite' + // statement, then skip security/privilege checks. + // Checks will be done when the corresponding insert node is processed. + NABoolean noSecurityCheck_; }; +/////////////////////////////////////////////////////////// +// ExeUtilHiveTruncate +/////////////////////////////////////////////////////////// class ExeUtilHiveTruncate : public ExeUtilExpr { public: - ExeUtilHiveTruncate(const CorrName &name, - ConstStringList * pl, + ExeUtilHiveTruncate(CorrName &name, + NAString &hiveTableName, + NAString &hiveTruncQuery, CollHeap *oHeap = CmpCommon::statementHeap()) : ExeUtilExpr(HIVE_TRUNCATE_, name, NULL, NULL, NULL, CharInfo::UnknownCharSet, oHeap), - pl_(pl), suppressModCheck_(FALSE), dropTableOnDealloc_(FALSE) + hiveTableName_(hiveTableName), + hiveTruncQuery_(hiveTruncQuery), + dropTableOnDealloc_(FALSE), + noSecurityCheck_(FALSE), + hiveExternalTable_(FALSE), + ifExist_(FALSE), + tableNotExist_(FALSE) { } virtual NABoolean isExeUtilQueryType() { return TRUE; } @@ -1103,50 +1121,49 @@ public: virtual RelExpr * bindNode(BindWA *bindWAPtr); - virtual RelExpr * preCodeGen(Generator * generator, - const ValueIdSet & externalInputs, - ValueIdSet &pulledNewInputs); - // method to do code generation virtual short codeGen(Generator*); + ExplainTuple *addSpecificExplainInfo(ExplainTupleMaster *explainTuple, + ComTdb * tdb, + Generator *generator); + virtual NABoolean aqrSupported() { return TRUE; } - const NAString &getHiveTableLocation() const - { - return hiveTableLocation_; - } + const NAString &getHiveTableName() const { return hiveTableName_; } + const NAString &getHiveTruncQuery() const { return hiveTruncQuery_; } - const NAString &getHiveHostName() const - { - return hiveHostName_; - } + NABoolean getDropTableOnDealloc() const { return dropTableOnDealloc_; } + NABoolean getNoSecurityCheck() const { return noSecurityCheck_; } + NABoolean getHiveExternalTable() const { return hiveExternalTable_; } + NABoolean getIfExist() const { return ifExist_; } + NABoolean getTableNotExist() const { return tableNotExist_; } - const Int32 getHiveHdfsPort() const - { - return hiveHdfsPort_; - } + void setDropTableOnDealloc(NABoolean v=TRUE) { dropTableOnDealloc_ = v; } + void setNoSecurityCheck(NABoolean v) { noSecurityCheck_ = v; } + void setHiveExternalTable(NABoolean v) { hiveExternalTable_ = v; } + void setIfExist(NABoolean v) { ifExist_ = v; } + void setTableNotExist(NABoolean v) { tableNotExist_ = v; } - ConstStringList* &partnList() { return pl_; } +private: - NABoolean getSuppressModCheck() const { return suppressModCheck_; } - NABoolean getDropTableOnDealloc() const { return dropTableOnDealloc_; } + NAString hiveTableName_; + NAString hiveTruncQuery_; + NABoolean dropTableOnDealloc_; - void setSuppressModCheck(NABoolean v=TRUE) { suppressModCheck_ = v; } - void setDropTableOnDealloc(NABoolean v=TRUE) { dropTableOnDealloc_ = v; } + // if this truncate node is added internally to process 'insert overwrite' + // statement, then skip security/privilege checks. + // Checks will be done when the corresponding insert node is processed. + NABoolean noSecurityCheck_; -private: - NAString hiveTableLocation_; - NAString hiveHostName_; - Int32 hiveHdfsPort_; + // TRUE: Hive External table. FALSE: Hive Managed table. + NABoolean hiveExternalTable_; - // timestamp of hiveTableLocation. - Int64 hiveModTS_; + // if 'if exist' clause is specified + NABoolean ifExist_; - // list of partitions to be truncated - ConstStringList * pl_; - NABoolean suppressModCheck_; - NABoolean dropTableOnDealloc_; + // if table does not exist + NABoolean tableNotExist_; }; class ExeUtilHiveQuery : public ExeUtilExpr http://git-wip-us.apache.org/repos/asf/trafodion/blob/2bba19ff/core/sql/optimizer/RelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExpr.cpp b/core/sql/optimizer/RelExpr.cpp index 44e0e90..669f806 100644 --- a/core/sql/optimizer/RelExpr.cpp +++ b/core/sql/optimizer/RelExpr.cpp @@ -13600,7 +13600,6 @@ Delete::Delete(const CorrName &name, TableDesc *tabId, OperatorTypeEnum otype, ConstStringList * csl, CollHeap *oHeap) : GenericUpdate(name,tabId,otype,child,newRecExpr,currOfCursorName,oHeap), - isFastDelete_(FALSE), csl_(csl),estRowsAccessed_(0) { setCacheableNode(CmpMain::BIND); @@ -13630,7 +13629,6 @@ RelExpr * Delete::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap) else result = (Delete *) derivedNode; - result->isFastDelete_ = isFastDelete_; result->csl() = csl(); result->setEstRowsAccessed(getEstRowsAccessed()); http://git-wip-us.apache.org/repos/asf/trafodion/blob/2bba19ff/core/sql/optimizer/RelFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelFastTransport.cpp b/core/sql/optimizer/RelFastTransport.cpp index 9981b6c..7365632 100644 --- a/core/sql/optimizer/RelFastTransport.cpp +++ b/core/sql/optimizer/RelFastTransport.cpp @@ -139,18 +139,85 @@ RelExpr *FastExtract::makeFastExtractTree( unloadRelExpr->setIsMainQueryOperator(calledFromBinder); result = unloadRelExpr; - if (overwriteTable) + // keeping older Hive Truncate around for time being. + // Once newer method is tested, legacy will be removed. + NABoolean legacyHiveTruncate = FALSE; + char * leg = getenv("TRUNC_LEGACY"); + if (leg) + legacyHiveTruncate = TRUE; + if (overwriteTable && legacyHiveTruncate) { + ExeUtilHiveTruncateLegacy *trunc = new (bindWA->wHeap()) + ExeUtilHiveTruncateLegacy(tableDesc->getCorrNameObj(), + NULL, + bindWA->wHeap()); + trunc->setNoSecurityCheck(TRUE); + + RelExpr * newRelExpr = trunc; + + if (tempTableForCSE) + { + trunc->setSuppressModCheck(); + + // This table gets created at compile time, unlike most + // other tables. It gets dropped when the statement is + // deallocated. Note that there are three problems: + // a) Statement gets never executed + // b) Process exits before deallocating the statement + // c) Statement gets deallocated, then gets executed again + // + // Todo: CSE: Handle these issues. + // Cases a) and b) are handled like volatile tables, there + // is a cleanup mechanism. + // Case c) gets handled by AQR. + trunc->setDropTableOnDealloc(); + } + + if (calledFromBinder) + //new root to prevent error 4056 when binding + newRelExpr = new (bindWA->wHeap()) RelRoot(newRelExpr); + else + // this node must be bound, even outside the binder, + // to set some values + newRelExpr = newRelExpr->bindNode(bindWA); + + Union *blockedUnion = new (bindWA->wHeap()) Union(newRelExpr, result); + + blockedUnion->setBlockedUnion(); + blockedUnion->setSerialUnion(); + result = blockedUnion; + } + + if (overwriteTable && (NOT legacyHiveTruncate)) + { + NAString hiveName = ComConvertTrafHiveNameToNativeHiveName + (tableDesc->getCorrNameObj().getQualifiedNameObj().getCatalogName(), + tableDesc->getCorrNameObj().getQualifiedNameObj().getSchemaName(), + tableDesc->getCorrNameObj().getQualifiedNameObj().getObjectName()); + + if (hiveName.isNull()) + { + *CmpCommon::diags() + << DgSqlCode(-3242) + << DgString0("Invalid Hive name specified."); + bindWA->setErrStatus(); + return NULL; + } + + NAString hiveTruncQuery("truncate table "); + hiveTruncQuery += hiveName; + ExeUtilHiveTruncate *trunc = new (bindWA->wHeap()) ExeUtilHiveTruncate(tableDesc->getCorrNameObj(), - NULL, + hiveName, + hiveTruncQuery, bindWA->wHeap()); + trunc->setNoSecurityCheck(TRUE); + RelExpr * newRelExpr = trunc; if (tempTableForCSE) { - trunc->setSuppressModCheck(); - // This table gets created at compile time, unlike most // other tables. It gets dropped when the statement is // deallocated. Note that there are three problems: http://git-wip-us.apache.org/repos/asf/trafodion/blob/2bba19ff/core/sql/optimizer/RelUpdate.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelUpdate.h b/core/sql/optimizer/RelUpdate.h index 1e0b9cd..a88aa64 100644 --- a/core/sql/optimizer/RelUpdate.h +++ b/core/sql/optimizer/RelUpdate.h @@ -1580,17 +1580,6 @@ public: const QualifiedName &subjectTable, UpdateColumns *updateCols = NULL); - void setIsFastDelete(NABoolean v) - { - isFastDelete_ = v; - } - NABoolean isFastDelete() - { - return isFastDelete_; - } - - // ValueIdList &lobDeleteExpr() { return lobDeleteExpr_; } - ConstStringList* &csl() { return csl_; } inline const CostScalar getEstRowsAccessed() const @@ -1599,10 +1588,6 @@ public: inline void setEstRowsAccessed(CostScalar r) { estRowsAccessed_ = r; } private: - NABoolean isFastDelete_; - - // ValueIdList lobDeleteExpr_; - ConstStringList * csl_; // Estimated number of rows accessed by Delete operator. CostScalar estRowsAccessed_; http://git-wip-us.apache.org/repos/asf/trafodion/blob/2bba19ff/core/sql/parser/sqlparser.y ---------------------------------------------------------------------- diff --git a/core/sql/parser/sqlparser.y b/core/sql/parser/sqlparser.y index 9f58820..968c096 100755 --- a/core/sql/parser/sqlparser.y +++ b/core/sql/parser/sqlparser.y @@ -2858,9 +2858,8 @@ static void enableMakeQuotedStringISO88591Mechanism() %type <pSchemaName> optional_from_schema %type <stringval> get_statistics_optional_options -%type <corrName> truncate_table_name -%type <relx> exe_util_fast_delete -%type <longint> purgedata_options +%type <ptr_placeholder> truncate_table_name +%type <relx> truncate_table %type <relx> exe_util_get_metadata_info %type <relx> exe_util_get_version_info @@ -14850,7 +14849,7 @@ interactive_query_expression: { $$ = finalize($1); } - | exe_util_fast_delete + | truncate_table { $$ = finalize($1); } @@ -17865,13 +17864,25 @@ optional_mt_options : QUOTED_STRING } /* type corrName */ -truncate_table_name : TOK_PURGEDATA table_name +truncate_table_name : TOK_PURGEDATA optional_if_exists_clause table_name { - $$ = $2; + $$ = new(PARSERHEAP()) + PtrPlaceHolder($3, + ($2 ? new(PARSERHEAP()) NAString() + : NULL)); } - | TOK_TRUNCATE table_name + | TOK_TRUNCATE optional_if_exists_clause { - $$ = $2; + SqlParser_CurrentParser->hiveDDLInfo_-> + setValues(TRUE, StmtDDLonHiveObjects::TRUNCATE_, StmtDDLonHiveObjects::TABLE_, $2); + } + ddl_qualified_name + { + CorrName * cn = new(PARSERHEAP()) CorrName(*$4, PARSERHEAP()); + $$ = new(PARSERHEAP()) + PtrPlaceHolder(cn, + ($2 ? new(PARSERHEAP()) NAString() + : NULL)); } | TOK_TRUNCATE TOK_TABLE optional_if_exists_clause { @@ -17880,13 +17891,16 @@ truncate_table_name : TOK_PURGEDATA table_name } ddl_qualified_name { - $$ = new (PARSERHEAP()) CorrName(*$5, PARSERHEAP()); + CorrName * cn = new(PARSERHEAP()) CorrName(*$5, PARSERHEAP()); + $$ = new(PARSERHEAP()) + PtrPlaceHolder(cn, + ($3 ? new(PARSERHEAP()) NAString() + : NULL)); } - -exe_util_fast_delete : truncate_table_name purgedata_options + +truncate_table : truncate_table_name { - short noLog = ($2 & 0x1) != 0; - short ignoreTrigger = ($2 & 0x2) != 0; + PtrPlaceHolder *pph = $1; CharInfo::CharSet stmtCharSet = CharInfo::UnknownCharSet; NAString * stmt = getSqlStmtStr ( stmtCharSet @@ -17897,31 +17911,20 @@ exe_util_fast_delete : truncate_table_name purgedata_options *SqlParser_Diags << DgSqlCode(-3406); YYERROR; } - $$ = new (PARSERHEAP()) - ExeUtilFastDelete(CorrName(*$1, PARSERHEAP()), - NULL, - (char*)stmt->data(), - stmtCharSet, - FALSE, - noLog, - ignoreTrigger, - TRUE, - PARSERHEAP()); - - delete $1; - } -purgedata_options : /*empty*/ { $$ = 0; } - | TOK_NOLOG { $$ = 1; } - | TOK_IGNORE_TRIGGER { $$ = 2; } - | TOK_NOLOG TOK_IGNORE_TRIGGER { $$ = 3; } - | TOK_IGNORE_TRIGGER TOK_NOLOG { $$ = 3; } - | TOK_WAITEDIO { $$ = 4; } - | TOK_NOLOG TOK_IGNORE_TRIGGER TOK_WAITEDIO { $$ = 7; } - | TOK_IGNORE_TRIGGER TOK_NOLOG TOK_WAITEDIO { $$ = 7; } - | TOK_WAITEDIO TOK_NOLOG TOK_IGNORE_TRIGGER { $$ = 7; } - | TOK_WAITEDIO TOK_IGNORE_TRIGGER TOK_NOLOG { $$ = 7; } - | TOK_NOLOG TOK_WAITEDIO TOK_IGNORE_TRIGGER { $$ = 7; } - | TOK_IGNORE_TRIGGER TOK_WAITEDIO TOK_NOLOG { $$ = 7; } + CorrName *cn = (CorrName*) pph->ptr1_; + NABoolean ifExists = (pph->ptr2_ != NULL); + + DDLExpr * ddlExpr = new(PARSERHEAP()) + DDLExpr(NULL, + (char*)stmt->data(), + CharInfo::UnknownCharSet, + CmpCommon::statementHeap()); + ddlExpr->setPurgedata(TRUE); + ddlExpr->setPurgedataTableName(*cn); + ddlExpr->setPurgedataIfExists(ifExists); + + $$ = ddlExpr; + } exe_util_aqr: TOK_GET TOK_ALL TOK_AQR TOK_ENTRIES { @@ -19575,10 +19578,10 @@ Rest_Of_insert_statement : no_check_log no_rollback TOK_INTO table_name query_ex $5); ((Insert*)$$)->setOverwriteHiveTable(TRUE); - + delete $3; } - + | no_check_log no_rollback TOK_INTO table_name '(' '*' ')' query_expression order_by_clause access_type optional_limit_spec { if (!finalizeAccessOptions($8, $10)) YYERROR; @@ -20637,76 +20640,8 @@ delete_statement : delete_start_tokens where_clause } } -delete_statement : TOK_DELETE TOK_DATA TOK_FROM table_name - { - if (CmpCommon::getDefault(FAST_DELETE) == DF_OFF) - { - YYERROR; - } - - CharInfo::CharSet stmtCharSet = CharInfo::UnknownCharSet; - NAString * stmt = getSqlStmtStr ( stmtCharSet // out - CharInfo::CharSet & - , PARSERHEAP() // in - NAMemory * heapUsedForOutputBuffers - ); - // If we can not get a variable-width multi-byte or single-byte string here, report error - if ( stmt == NULL ) - { - *SqlParser_Diags << DgSqlCode(-3406); - YYERROR; - } - $$ = new (PARSERHEAP()) - ExeUtilFastDelete(CorrName(*$4, PARSERHEAP()), - NULL, - (char*)stmt->data(), - stmtCharSet, - FALSE, - FALSE, FALSE, - FALSE, - PARSERHEAP()); - - delete $4; - } - /* type relx */ -delete_statement : TOK_DELETE TOK_USING TOK_PURGEDATA TOK_FROM table_name - { - Scan * inputScan = - new (PARSERHEAP()) Scan(CorrName(*$5, PARSERHEAP())); - - Delete *del = new (PARSERHEAP()) - Delete(CorrName(*$5, PARSERHEAP()), - NULL, - REL_UNARY_DELETE, - inputScan, - NULL); - - del->setIsFastDelete(TRUE); - - delete $5; - - $$ = del; - } - - | TOK_DELETE TOK_NO TOK_PURGEDATA TOK_FROM table_name - { - Scan * inputScan = - new (PARSERHEAP()) Scan(CorrName(*$5, PARSERHEAP())); - - Delete *del = new (PARSERHEAP()) - Delete(CorrName(*$5, PARSERHEAP()), - NULL, - REL_UNARY_DELETE, - inputScan, - NULL); - - del->setIsFastDelete(FALSE); - del->setNoIMneeded(TRUE); - - delete $5; - - $$ = del; - } - | TOK_DELETE no_check_log TOK_WITH TOK_NO TOK_ROLLBACK TOK_FROM table_name optimizer_hint where_clause +delete_statement : TOK_DELETE no_check_log TOK_WITH TOK_NO TOK_ROLLBACK TOK_FROM table_name optimizer_hint where_clause { Scan * inputScan = new (PARSERHEAP()) Scan(CorrName(*$7, PARSERHEAP())); http://git-wip-us.apache.org/repos/asf/trafodion/blob/2bba19ff/core/sql/regress/hive/EXPECTED005 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED005 b/core/sql/regress/hive/EXPECTED005 index 298547b..2b4490f 100644 --- a/core/sql/regress/hive/EXPECTED005 +++ b/core/sql/regress/hive/EXPECTED005 @@ -72,7 +72,7 @@ *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1528150079000, failedModTS = 1528150239571, failedLoc = hdfs://localhost:36000/user/trafodion/hive/exttables/customer_ddl +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1529149450876, failedModTS = 1529149578260, failedLoc = hdfs://localhost:24200/user/trafodion/hive/exttables/customer_ddl C_PREFERRED_CUST_FLAG (EXPR) ------------------------- -------------------- @@ -108,7 +108,7 @@ Y 9525 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1528150079000, failedModTS = 1528150239571, failedLoc = hdfs://localhost:36000/user/trafodion/hive/exttables/customer_ddl +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1529149450876, failedModTS = 1529149578260, failedLoc = hdfs://localhost:24200/user/trafodion/hive/exttables/customer_ddl C_PREFERRED_CUST_FLAG (EXPR) ------------------------- -------------------- @@ -177,7 +177,7 @@ Y 9525 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1528150242355, failedModTS = 1528150250981, failedLoc = hdfs://localhost:36000/user/hive/warehouse/newtable +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1529149581595, failedModTS = 1529149591176, failedLoc = hdfs://localhost:24200/user/hive/warehouse/newtable A ------------------------- @@ -227,7 +227,7 @@ xyz *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1528150239571, failedModTS = 1528150270615, failedLoc = hdfs://localhost:36000/user/trafodion/hive/exttables/customer_ddl +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1529149578260, failedModTS = 1529149609405, failedLoc = hdfs://localhost:24200/user/trafodion/hive/exttables/customer_ddl C_PREFERRED_CUST_FLAG (EXPR) ------------------------- -------------------- @@ -242,7 +242,7 @@ Y 18984 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1528150239571, failedModTS = 1528150270615, failedLoc = hdfs://localhost:36000/user/trafodion/hive/exttables/customer_ddl +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1529149578260, failedModTS = 1529149609405, failedLoc = hdfs://localhost:24200/user/trafodion/hive/exttables/customer_ddl C_PREFERRED_CUST_FLAG (EXPR) ------------------------- -------------------- @@ -298,7 +298,7 @@ Y 18984 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1528150267991, failedModTS = 1528150279764, failedLoc = hdfs://localhost:36000/user/hive/warehouse/newtable +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1529149606436, failedModTS = 1529149616936, failedLoc = hdfs://localhost:24200/user/hive/warehouse/newtable A B ----------- ------------------------- @@ -617,18 +617,18 @@ C1 C2 C3 C4 C --- 1 row(s) selected. >>load with continue on error into trafodion.seabase.traf_tbl_bad select * >>from tbl_bad; Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD -Task: CLEANUP Status: Started Time: 2018-06-04 22:12:37.338586 -Task: CLEANUP Status: Ended Time: 2018-06-04 22:12:37.349263 -Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.011 -Task: LOADING DATA Status: Started Time: 2018-06-04 22:12:37.349352 +Task: CLEANUP Status: Started Time: 2018-06-16 11:47:50.919693 +Task: CLEANUP Status: Ended Time: 2018-06-16 11:47:50.927679 +Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.008 +Task: LOADING DATA Status: Started Time: 2018-06-16 11:47:50.927713 Rows Processed: 8 Error Rows: 5 -Task: LOADING DATA Status: Ended Time: 2018-06-04 22:12:37.525012 -Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.176 -Task: COMPLETION Status: Started Time: 2018-06-04 22:12:37.525050 +Task: LOADING DATA Status: Ended Time: 2018-06-16 11:47:51.85803 +Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.158 +Task: COMPLETION Status: Started Time: 2018-06-16 11:47:51.85854 Rows Loaded: 3 -Task: COMPLETION Status: Ended Time: 2018-06-04 22:12:37.888170 -Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.363 +Task: COMPLETION Status: Ended Time: 2018-06-16 11:47:52.476978 +Task: COMPLETION Status: Ended Elapsed Time: 00:00:01.391 --- 3 row(s) loaded. >>select count(*) from trafodion.seabase.traf_tbl_bad; @@ -644,19 +644,19 @@ Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.363 --- 3 row(s) deleted. >>load with log error rows into trafodion.seabase.traf_tbl_bad select * from >>tbl_bad; Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD -Task: CLEANUP Status: Started Time: 2018-06-04 22:12:38.932654 -Task: CLEANUP Status: Ended Time: 2018-06-04 22:12:38.947277 -Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.015 - Logging Location: /user/trafodion/bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20180604_221238 -Task: LOADING DATA Status: Started Time: 2018-06-04 22:12:38.947419 +Task: CLEANUP Status: Started Time: 2018-06-16 11:47:53.526469 +Task: CLEANUP Status: Ended Time: 2018-06-16 11:47:53.536440 +Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.010 + Logging Location: /user/trafodion/bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20180616_114753 +Task: LOADING DATA Status: Started Time: 2018-06-16 11:47:53.536510 Rows Processed: 8 Error Rows: 5 -Task: LOADING DATA Status: Ended Time: 2018-06-04 22:12:39.164857 -Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.217 -Task: COMPLETION Status: Started Time: 2018-06-04 22:12:39.164924 +Task: LOADING DATA Status: Ended Time: 2018-06-16 11:47:53.760748 +Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.224 +Task: COMPLETION Status: Started Time: 2018-06-16 11:47:53.760793 Rows Loaded: 3 -Task: COMPLETION Status: Ended Time: 2018-06-04 22:12:39.538579 -Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.374 +Task: COMPLETION Status: Ended Time: 2018-06-16 11:47:56.167112 +Task: COMPLETION Status: Ended Elapsed Time: 00:00:02.406 --- 3 row(s) loaded. >>select count(*) from trafodion.seabase.traf_tbl_bad; @@ -669,19 +669,19 @@ Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.374 --- 1 row(s) selected. >>load with log error rows to '/user/trafodion/bulkload/logs/TEST005' into >>trafodion.seabase.traf_tbl_bad select * from tbl_bad; Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD -Task: CLEANUP Status: Started Time: 2018-06-04 22:12:40.528226 -Task: CLEANUP Status: Ended Time: 2018-06-04 22:12:40.541034 -Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.013 - Logging Location: /user/trafodion/bulkload/logs/TEST005/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20180604_221240 -Task: LOADING DATA Status: Started Time: 2018-06-04 22:12:40.541082 +Task: CLEANUP Status: Started Time: 2018-06-16 11:47:57.157930 +Task: CLEANUP Status: Ended Time: 2018-06-16 11:47:57.165444 +Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.008 + Logging Location: /user/trafodion/bulkload/logs/TEST005/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20180616_114757 +Task: LOADING DATA Status: Started Time: 2018-06-16 11:47:57.165499 Rows Processed: 8 Error Rows: 5 -Task: LOADING DATA Status: Ended Time: 2018-06-04 22:12:40.732369 -Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.191 -Task: COMPLETION Status: Started Time: 2018-06-04 22:12:40.732402 +Task: LOADING DATA Status: Ended Time: 2018-06-16 11:47:57.352294 +Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.187 +Task: COMPLETION Status: Started Time: 2018-06-16 11:47:57.352327 Rows Loaded: 3 -Task: COMPLETION Status: Ended Time: 2018-06-04 22:12:41.125846 -Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.393 +Task: COMPLETION Status: Ended Time: 2018-06-16 11:47:57.916072 +Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.564 --- 3 row(s) loaded. >>select count(*) from trafodion.seabase.traf_tbl_bad; @@ -697,10 +697,10 @@ Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.393 --- 6 row(s) deleted. >>load with stop after 3 error rows into trafodion.seabase.traf_tbl_bad select >>* from tbl_bad; Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD -Task: CLEANUP Status: Started Time: 2018-06-04 22:12:42.147737 -Task: CLEANUP Status: Ended Time: 2018-06-04 22:12:42.157127 +Task: CLEANUP Status: Started Time: 2018-06-16 11:47:58.929152 +Task: CLEANUP Status: Ended Time: 2018-06-16 11:47:58.937786 Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.009 -Task: LOADING DATA Status: Started Time: 2018-06-04 22:12:42.157174 +Task: LOADING DATA Status: Started Time: 2018-06-16 11:47:58.937815 *** ERROR[8113] The maximum number of error rows is exceeded. @@ -715,11 +715,11 @@ Task: LOADING DATA Status: Started Time: 2018-06-04 22:12:42.157174 --- 1 row(s) selected. >>load with log error rows, stop after 3 error rows into >>trafodion.seabase.traf_tbl_bad select * from tbl_bad; Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD -Task: CLEANUP Status: Started Time: 2018-06-04 22:12:43.279600 -Task: CLEANUP Status: Ended Time: 2018-06-04 22:12:43.289771 -Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.010 - Logging Location: /user/trafodion/bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20180604_221243 -Task: LOADING DATA Status: Started Time: 2018-06-04 22:12:43.289816 +Task: CLEANUP Status: Started Time: 2018-06-16 11:48:00.37617 +Task: CLEANUP Status: Ended Time: 2018-06-16 11:48:00.45699 +Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.008 + Logging Location: /user/trafodion/bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20180616_114800 +Task: LOADING DATA Status: Started Time: 2018-06-16 11:48:00.45732 *** ERROR[8113] The maximum number of error rows is exceeded. @@ -809,7 +809,7 @@ Task: LOADING DATA Status: Started Time: 2018-06-04 22:12:43.289816 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1528150390948, failedModTS = 1528150393514, failedLoc = hdfs://localhost:36000/user/hive/warehouse/thive +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1529149699813, failedModTS = 1529149706160, failedLoc = hdfs://localhost:24200/user/hive/warehouse/thive A ----------- @@ -866,16 +866,16 @@ A B --- SQL operation complete. >>select * from hive.hive.thive; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1529149722198, failedModTS = 1529149723553, failedLoc = hdfs://localhost:24200/user/hive/warehouse/thive + --- 0 row(s) selected. >>insert into hive.hive.thive values (10, 20); --- 1 row(s) inserted. >>select * from hive.hive.thive; -*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. - -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1528150404696, failedModTS = 1528150406756, failedLoc = hdfs://localhost:36000/user/hive/warehouse/thive - A B ----------- ----------- @@ -890,6 +890,10 @@ A B --- SQL operation complete. >>select * from hive.hive.thive; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1529149724776, failedModTS = 1529149727335, failedLoc = hdfs://localhost:24200/user/hive/warehouse/thive + --- 0 row(s) selected. >> >>-- truncate of partitioned hive table @@ -962,7 +966,7 @@ t005part.a t005part.b t005part.c >>-- should return error >>truncate table hive.hive.t005part partition (b=10,c=13); -*** ERROR[1214] Error encountered when executing HiveQL statement "truncate table `default`.T005PART partition (b=10,c=13)". Details: org.apache.hive.service.cli.HiveSQLException: Error while compiling statement: FAILED: SemanticException [Error 10006]: Partition not found {b=10, c=13} +*** ERROR[1214] Error encountered when executing HiveQL statement "TRUNCATE TABLE `default`.T005PART partition (b=10,c=13)". Details: org.apache.hive.service.cli.HiveSQLException: Error while compiling statement: FAILED: SemanticException [Error 10006]: Partition not found {b=10, c=13} --- SQL operation failed with errors. >> @@ -971,14 +975,13 @@ t005part.a t005part.b t005part.c *** ERROR[3242] This statement is not supported. Reason: Purgedata is not allowed for Hive tables. Use 'Truncate Table' command. -*** ERROR[8822] The statement was not prepared. - +--- SQL operation failed with errors. >> >>-- tests for hive insert error modes >>invoke hive.hive.thive_insert_smallint; -- Definition of hive table HIVE.HIVE.THIVE_INSERT_SMALLINT --- Definition current Mon Jun 4 22:16:40 2018 +-- Definition current Sat Jun 16 11:51:20 2018 ( A SMALLINT @@ -999,7 +1002,7 @@ CREATE TABLE HIVE.HIVE.THIVE_INSERT_SMALLINT /* Trafodion DDL */ REGISTER /*INTERNAL*/ HIVE TABLE HIVE.HIVE.THIVE_INSERT_SMALLINT; -/* ObjectUID = 7976600667680320368 */ +/* ObjectUID = 7436169711789392123 */ --- SQL operation complete. >> @@ -1012,6 +1015,10 @@ REGISTER /*INTERNAL*/ HIVE TABLE HIVE.HIVE.THIVE_INSERT_SMALLINT; >>insert into hive.hive.thive_insert_smallint select * from +> (values (10), (11111111), (21), (22222222)); +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1529149537134, failedModTS = 1529149881411, failedLoc = hdfs://localhost:24200/user/hive/warehouse/thive_insert_smallint + --- 4 row(s) inserted. >>select * from hive.hive.thive_insert_smallint; @@ -1049,6 +1056,10 @@ A >>insert into hive.hive.thive_insert_smallint select * from +> (values (10), (11111111), (21), (22222222)); +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1529149883896, failedModTS = 1529149886157, failedLoc = hdfs://localhost:24200/user/hive/warehouse/thive_insert_smallint + --- 2 row(s) inserted. >>select * from hive.hive.thive_insert_smallint; @@ -1088,7 +1099,7 @@ A >>invoke hive.hive.thive_insert_varchar; -- Definition of hive table HIVE.HIVE.THIVE_INSERT_VARCHAR --- Definition current Mon Jun 4 22:17:04 2018 +-- Definition current Sat Jun 16 11:51:32 2018 ( A VARCHAR(1 CHAR) CHARACTER SET UTF8 COLLATE @@ -1113,7 +1124,7 @@ CREATE TABLE HIVE.HIVE.THIVE_INSERT_VARCHAR /* Trafodion DDL */ REGISTER /*INTERNAL*/ HIVE TABLE HIVE.HIVE.THIVE_INSERT_VARCHAR; -/* ObjectUID = 7976600667680321080 */ +/* ObjectUID = 7436169711789392374 */ --- SQL operation complete. >>cqd hive_insert_error_mode '1'; @@ -1124,6 +1135,10 @@ REGISTER /*INTERNAL*/ HIVE TABLE HIVE.HIVE.THIVE_INSERT_VARCHAR; --- SQL operation complete. >>insert into hive.hive.thive_insert_varchar values ('abcddcba','efghijkl'); +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1529149539689, failedModTS = 1529149893202, failedLoc = hdfs://localhost:24200/user/hive/warehouse/thive_insert_varchar + --- 1 row(s) inserted. >> >>cqd hive_max_string_length_in_bytes '20'; http://git-wip-us.apache.org/repos/asf/trafodion/blob/2bba19ff/core/sql/regress/hive/EXPECTED008 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED008 b/core/sql/regress/hive/EXPECTED008 index ce76529..3d07895 100644 --- a/core/sql/regress/hive/EXPECTED008 +++ b/core/sql/regress/hive/EXPECTED008 @@ -872,9 +872,10 @@ A B >>-- next truncate should return error >>truncate table t00807; -*** ERROR[1388] Table sch008.t00807 does not exist in Hive Metadata. +*** ERROR[1388] Object HIVE.SCH008.T00807 does not exist in Hive Metadata. + +*** ERROR[8822] The statement was not prepared. ---- SQL operation failed with errors. >>truncate table if exists t00807; --- SQL operation complete. http://git-wip-us.apache.org/repos/asf/trafodion/blob/2bba19ff/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp b/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp index 9e6e464..f306ce3 100644 --- a/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp +++ b/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp @@ -8741,7 +8741,31 @@ void CmpSeabaseDDL::purgedataHbaseTable(DDLExpr * ddlExpr, tableName.getSchemaNamePart().getInternalName(), tableName.getCatalogNamePart().getInternalName()); cn.setSpecialType(purgedataTableName); + + if (cn.isHive()) + { + *CmpCommon::diags() << DgSqlCode(-3242) + << DgString0("Purgedata is not allowed for Hive tables. Use 'Truncate Table' command."); + return; + } + + if (cn.isHbase()) + { + *CmpCommon::diags() << DgSqlCode(-3242) + << DgString0("Purgedata is not allowed for HBase tables."); + return; + } + NATable *naTable = bindWA.getNATable(cn); + + // if table doesn't exist and 'if exists' clause is specified, return. + if (ddlExpr->purgedataIfExists() && (! naTable)) + { + bindWA.resetErrStatus(); + CmpCommon::diags()->clear(); + return; + } + if (naTable == NULL || bindWA.errStatus()) { processReturn(); http://git-wip-us.apache.org/repos/asf/trafodion/blob/2bba19ff/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp index 3e9082d..815381d 100644 --- a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp +++ b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp @@ -11351,81 +11351,6 @@ void CmpSeabaseDDL::regOrUnregNativeObject( return; } -static short processHiveTruncate(StmtDDLonHiveObjects * hddl, - NATable * naTable, - NAString &extObjectName) -{ - // A Hive table can be an External or Managed table. - // Currently, an External Hive table cannot be truncated. - // Maybe some future Hive version will allow that. - // Temporarily change the table attribute to be Managed, - // truncate the table and then change it back to be External. - NABoolean tableWasAltered = FALSE; - NAString alterStmt; - if ((hddl->getOper() == StmtDDLonHiveObjects::TRUNCATE_) && - (naTable && naTable->isHiveExternalTable())) - { - // make the table managed table before truncate - alterStmt = "alter table " + extObjectName + " set tblproperties ('EXTERNAL'='False')"; - if (HiveClient_JNI::executeHiveSQL(alterStmt.data()) != HVC_OK) - { - // alter failed - *CmpCommon::diags() << DgSqlCode(-1214) - << DgString0(getSqlJniErrorStr()) - << DgString1(hddl->getHiveDDL()); - - return -1; - } - - tableWasAltered = TRUE; - - // alter stmt to switch the table back to external type - alterStmt = "alter table " + extObjectName + " set tblproperties ('EXTERNAL'='TRUE')"; - } - - // execute the hive DDL statement. - if (HiveClient_JNI::executeHiveSQL(hddl->getHiveDDL().data()) != HVC_OK) - { - if (tableWasAltered) - { - // table was altered to Managed. Alter it back to External. - if (HiveClient_JNI::executeHiveSQL(alterStmt.data()) != HVC_OK) - { - // alter itself failed. Return error. - *CmpCommon::diags() << DgSqlCode(-1214) - << DgString0(getSqlJniErrorStr()) - << DgString1(hddl->getHiveDDL()); - - return -1; - } - } - - *CmpCommon::diags() << DgSqlCode(-1214) - << DgString0(getSqlJniErrorStr()) - << DgString1(hddl->getHiveDDL()); - - return -1; - } - - if ((hddl->getOper() == StmtDDLonHiveObjects::TRUNCATE_) && - (naTable && naTable->isHiveExternalTable()) && - (tableWasAltered)) - { - // table was altered to Managed. Alter it back to External. - if (HiveClient_JNI::executeHiveSQL(alterStmt.data()) != HVC_OK) - { - // alter itself failed. Return error. - *CmpCommon::diags() << DgSqlCode(-1214) - << DgString0(getSqlJniErrorStr()) - << DgString1(hddl->getHiveDDL()); - - return -1; - } - } - - return 0; -} - static void processPassthruHiveDDL(StmtDDLonHiveObjects * hddl) { NAString hiveQuery(hddl->getHiveDDL()); @@ -11734,22 +11659,14 @@ void CmpSeabaseDDL::processDDLonHiveObjects(StmtDDLonHiveObjects * hddl, } } // register this object - if (hddl->getOper() == StmtDDLonHiveObjects::TRUNCATE_) + // execute the hive DDL statement. + if (HiveClient_JNI::executeHiveSQL(hddl->getHiveDDL().data()) != HVC_OK) { - if (processHiveTruncate(hddl, naTable, extObjectName)) - goto label_error; - } - else - { - // execute the hive DDL statement. - if (HiveClient_JNI::executeHiveSQL(hddl->getHiveDDL().data()) != HVC_OK) - { - *CmpCommon::diags() << DgSqlCode(-1214) - << DgString0(getSqlJniErrorStr()) - << DgString1(hddl->getHiveDDL()); - - goto label_error; - } + *CmpCommon::diags() << DgSqlCode(-1214) + << DgString0(getSqlJniErrorStr()) + << DgString1(hddl->getHiveDDL()); + + goto label_error; } endXnIfStartedHere(&cliInterface, xnWasStartedHere, 0); @@ -11831,46 +11748,16 @@ NABoolean CmpSeabaseDDL::setupQueryTreeForHiveDDL( } NAString newHiveName; - if (con.getCatalogNamePartAsAnsiString(TRUE).compareTo(HIVE_SYSTEM_CATALOG, NAString::ignoreCase) != 0) + newHiveName = ComConvertTrafHiveNameToNativeHiveName + (con.getCatalogNamePartAsAnsiString(TRUE), + con.getSchemaNamePartAsAnsiString(TRUE), + (type != StmtDDLonHiveObjects::SCHEMA_ ? + con.getObjectNamePartAsAnsiString(TRUE) : NAString(""))); + if (newHiveName.isNull()) { - // should never reach here. Parser should have validated that the - // name is a hive name. - // Throw an assertion if it does. PARSERASSERT(1); } - // Original hive name specified in the query may have any of the following - // forms after they are fully qualified: - // hive.hive.t, hive.`default`.t, hive.hivesch.t, hive.hivesch - // These names are valid in traf environment only and are used to determine - // if hive ddl is being processed. - // - // Replace them in the query by equivalent hive names: - // t, `default`.t, hivesch.t, hivesch - // - if (con.getSchemaNamePartAsAnsiString(TRUE).compareTo(HIVE_DEFAULT_SCHEMA_EXE, NAString::ignoreCase) == 0) // matches 'default' - { - newHiveName += NAString("`") + con.getSchemaNamePartAsAnsiString(TRUE) + "`"; - if (type != StmtDDLonHiveObjects::SCHEMA_) - newHiveName += "."; - } - else if (con.getSchemaNamePartAsAnsiString(TRUE).compareTo(HIVE_SYSTEM_SCHEMA, NAString::ignoreCase) == 0) // matches 'hive' - { - // set fully qualified hive default schema name `default` - newHiveName += NAString("`default`"); - if (type != StmtDDLonHiveObjects::SCHEMA_) - newHiveName += "."; - } - else // user schema name - { - newHiveName += con.getSchemaNamePartAsAnsiString(1); - if (type != StmtDDLonHiveObjects::SCHEMA_) - newHiveName += "."; - } - - if (type != StmtDDLonHiveObjects::SCHEMA_) - newHiveName += con.getObjectNamePartAsAnsiString(TRUE); - // remove original name at hiveNamePos/hiveNameLen and replace with the // newly constructed name. if ((hiveDDL.length() > 0) && (hiveNameLen > 0)) @@ -11885,6 +11772,29 @@ NABoolean CmpSeabaseDDL::setupQueryTreeForHiveDDL( hiveDDL.remove(hiveDDL.length()-1); CmpCommon::diags()->clear(); + + DDLExpr * ddlExpr = NULL; + RelExpr * ddlExprRoot = NULL; + if (oper == StmtDDLonHiveObjects::TRUNCATE_) + { + NAString newHiveDDL("TRUNCATE TABLE "); + newHiveDDL += hiveDDL(hiveNamePos, hiveDDL.length() - hiveNamePos); + hiveDDL.clear(); + hiveDDL = newHiveDDL; + + CorrName cn(con.getObjectNamePartAsAnsiString(), + PARSERHEAP(), + con.getSchemaNamePartAsAnsiString(), + con.getCatalogNamePartAsAnsiString()); + ExeUtilHiveTruncate * ht = + new (PARSERHEAP()) ExeUtilHiveTruncate + (cn, newHiveName, hiveDDL, PARSERHEAP()); + + if (ifExistsOrNotExists) + ht->setIfExist(TRUE); + + ddlExprRoot = new(CmpCommon::statementHeap()) RelRoot(ht); + } // Construct DDL expr tree for regular query or query // explain/showplan/showshape/display. @@ -11902,10 +11812,8 @@ NABoolean CmpSeabaseDDL::setupQueryTreeForHiveDDL( // display: // same as regular query with displayTree flag set in RelRoot // - DDLExpr * ddlExpr = NULL; - RelExpr * ddlExprRoot = NULL; - if (NOT ((hiveDDLInfo->essd_ == Parser::HiveDDLInfo::SHOWPLAN_) || - (hiveDDLInfo->essd_ == Parser::HiveDDLInfo::SHOWSHAPE_))) + else if (NOT ((hiveDDLInfo->essd_ == Parser::HiveDDLInfo::SHOWPLAN_) || + (hiveDDLInfo->essd_ == Parser::HiveDDLInfo::SHOWSHAPE_))) { StmtDDLonHiveObjects * sdho = new (PARSERHEAP()) StmtDDLonHiveObjects(oper, type, http://git-wip-us.apache.org/repos/asf/trafodion/blob/2bba19ff/core/sql/sqlcomp/DefaultConstants.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h index 748ace0..e5ce78a 100644 --- a/core/sql/sqlcomp/DefaultConstants.h +++ b/core/sql/sqlcomp/DefaultConstants.h @@ -1817,8 +1817,6 @@ enum DefaultConstants GEN_UDRRS_SIZE_DOWN , GEN_UDRRS_SIZE_UP , - FAST_DELETE, - ALLOW_UNEXTERNALIZED_MAINTAIN_OPTIONS, EXE_MEMORY_AVAILABLE_IN_MB, // Mem size (MB) in ESP available for BMOs http://git-wip-us.apache.org/repos/asf/trafodion/blob/2bba19ff/core/sql/sqlcomp/nadefaults.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp index a0f245c..14b3ad9 100644 --- a/core/sql/sqlcomp/nadefaults.cpp +++ b/core/sql/sqlcomp/nadefaults.cpp @@ -1221,8 +1221,6 @@ SDDkwd__(EXE_DIAGNOSTIC_EVENTS, "OFF"), DDkwd__(FAKE_VOLUME_ASSIGNMENTS, "OFF"), DDui1__(FAKE_VOLUME_NUM_VOLUMES, "24"), - DDkwd__(FAST_DELETE, "OFF"), - // upper and lower limit (2,10) must be in sync with error values in //ExFastTransport.cpp DDkwd__(FAST_EXTRACT_DIAGS, "OFF"),
