Repository: incubator-trafodion Updated Branches: refs/heads/master 95723215c -> e014f8e60
JIRA TRAFODION-2022 Some more changes related to hive truncate -- support for 'TRUNCATE' command for hive tables -- support for truncate with PARTITION specification -- separation of traf purgedata and hive truncate code into different classes and tdbs -- some cleanup of obsolete older purgedata code -- new tests added to regress/hive/TEST005 Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/e7a1dc0d Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/e7a1dc0d Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/e7a1dc0d Branch: refs/heads/master Commit: e7a1dc0d46cea5e10cf93262f0e3d6154faffa96 Parents: 9dff321 Author: Anoop Sharma <[email protected]> Authored: Thu Jun 16 18:53:41 2016 +0000 Committer: Anoop Sharma <[email protected]> Committed: Thu Jun 16 18:53:41 2016 +0000 ---------------------------------------------------------------------- core/sql/bin/SqlciErrors.txt | 1 + core/sql/comexe/ComTdb.cpp | 8 ++ core/sql/comexe/ComTdb.h | 1 + core/sql/comexe/ComTdbExeUtil.cpp | 137 +++++++++++++++++---- core/sql/comexe/ComTdbExeUtil.h | 123 ++++++++++++------- core/sql/executor/ExComTdb.cpp | 9 +- core/sql/executor/ExExeUtil.h | 61 ++++++++- core/sql/executor/ExExeUtilMisc.cpp | 68 +++++++--- core/sql/exp/ExpErrorEnums.h | 4 +- core/sql/exp/ExpLOBaccess.cpp | 81 ++++++++---- core/sql/exp/ExpLOBaccess.h | 3 +- core/sql/generator/GenPreCode.cpp | 10 ++ core/sql/generator/GenRelExeUtil.cpp | 112 ++++++++++++++--- core/sql/optimizer/BindRelExpr.cpp | 15 +-- core/sql/optimizer/OptPhysRelExpr.cpp | 31 +---- core/sql/optimizer/RelExeUtil.cpp | 191 ++++++++++++++++------------- core/sql/optimizer/RelExeUtil.h | 94 ++++++++------ core/sql/parser/sqlparser.y | 37 ++++-- core/sql/regress/hive/EXPECTED005 | 99 +++++++++++++-- core/sql/regress/hive/TEST005 | 59 ++++++++- core/sql/sqlci/sqlci_lex.ll | 1 + core/sql/sqlci/sqlci_yacc.y | 2 + core/sql/sqlcomp/DefaultConstants.h | 6 - core/sql/sqlcomp/nadefaults.cpp | 10 -- 24 files changed, 819 insertions(+), 344 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/bin/SqlciErrors.txt ---------------------------------------------------------------------- diff --git a/core/sql/bin/SqlciErrors.txt b/core/sql/bin/SqlciErrors.txt index 0b96d26..0f19d9e 100644 --- a/core/sql/bin/SqlciErrors.txt +++ b/core/sql/bin/SqlciErrors.txt @@ -1492,6 +1492,7 @@ $1~String1 -------------------------------- 8031 HY018 99999 BEGINNER INFRM LOGONLY Server declined cancel request for query ID $0~String0. $1~String1. 8032 ZZZZZ 99999 ADVANCED MAJOR DBADMIN Explain information for the provided query id is not available in repository. 8033 ZZZZZ 99999 ADVANCED MAJOR DBADMIN Explain information is too large to be stored in repository. +8034 ZZZZZ 99999 ADVANCED MAJOR DBADMIN Truncation of hive table failed. $0~String0 8100 ZZZZZ 99999 BEGINNER MINOR LOGONLY Define $1~string0 does not exist 8101 23000 99999 BEGINNER MAJOR DBADMIN The operation is prevented by check constraint $0~ConstraintName on table $1~TableName. 8102 23000 99999 BEGINNER MAJOR DBADMIN The operation is prevented by a unique constraint. http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/comexe/ComTdb.cpp ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdb.cpp b/core/sql/comexe/ComTdb.cpp index dc8e5c4..2e2817c 100644 --- a/core/sql/comexe/ComTdb.cpp +++ b/core/sql/comexe/ComTdb.cpp @@ -757,6 +757,14 @@ NA_EIDPROC char *ComTdb::findVTblPtrCom(short classID) break; } + case ex_HIVE_TRUNCATE: + { +#pragma nowarn(1506) // warning elimination + GetVTblPtr(vtblptr,ComTdbExeUtilHiveTruncate); +#pragma warn(1506) // warning elimination + break; + } + case ex_GET_STATISTICS: { #pragma nowarn(1506) // warning elimination http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/comexe/ComTdb.h ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdb.h b/core/sql/comexe/ComTdb.h index 689bded..e0d39da 100644 --- a/core/sql/comexe/ComTdb.h +++ b/core/sql/comexe/ComTdb.h @@ -319,6 +319,7 @@ public: ex_ORC_AGGR = 150, ex_DDL_WITH_STATUS = 151, ex_GET_QID = 152, + ex_HIVE_TRUNCATE = 153, ex_LAST = 9999 // not used }; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/comexe/ComTdbExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbExeUtil.cpp b/core/sql/comexe/ComTdbExeUtil.cpp index 05cd4e5..0f22329 100644 --- a/core/sql/comexe/ComTdbExeUtil.cpp +++ b/core/sql/comexe/ComTdbExeUtil.cpp @@ -1185,12 +1185,7 @@ ComTdbExeUtilFastDelete::ComTdbExeUtilFastDelete( queue_index down, queue_index up, Lng32 num_buffers, - ULng32 buffer_size, - NABoolean isHiveTruncate, - char * hiveTableLocation, - char * hiveHostName, - Lng32 hivePortNum, - Int64 hiveModTS) + ULng32 buffer_size) : ComTdbExeUtil(ComTdbExeUtil::FAST_DELETE_, NULL, 0, (Int16)SQLCHARSETCODE_UNKNOWN, tableName, tableNameLen, @@ -1209,13 +1204,8 @@ ComTdbExeUtilFastDelete::ComTdbExeUtilFastDelete( numEsps_(numEsps), objectUID_(objectUID), numLOBs_(numLOBs), - lobNumArray_(lobNumArray), - hiveTableLocation_(hiveTableLocation), - hiveHdfsHost_(hiveHostName), - hiveHdfsPort_(hivePortNum), - hiveModTS_(hiveModTS) + lobNumArray_(lobNumArray) { - setIsHiveTruncate(isHiveTruncate); setNodeType(ComTdb::ex_FAST_DELETE); } @@ -1232,13 +1222,6 @@ Long ComTdbExeUtilFastDelete::pack(void * space) if (lobNumArray_) lobNumArray_.pack(space); - if (hiveTableLocation_) - hiveTableLocation_.pack(space); - - if (hiveHdfsHost_) - hiveHdfsHost_.pack(space); - - return ComTdbExeUtil::pack(space); } @@ -1255,12 +1238,6 @@ Lng32 ComTdbExeUtilFastDelete::unpack(void * base, void * reallocator) if(lobNumArray_.unpack(base)) return -1; - if(hiveTableLocation_.unpack(base)) - return -1; - - if(hiveHdfsHost_.unpack(base)) - return -1; - return ComTdbExeUtil::unpack(base, reallocator); } @@ -1305,6 +1282,7 @@ void ComTdbExeUtilFastDelete::displayContents(Space * space, space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(short)); } + } @@ -1318,6 +1296,115 @@ void ComTdbExeUtilFastDelete::displayContents(Space * space, /////////////////////////////////////////////////////////////////////////// // +// Methods for class ComTdbExeUtilHiveTruncate +// +/////////////////////////////////////////////////////////////////////////// +ComTdbExeUtilHiveTruncate::ComTdbExeUtilHiveTruncate( + char * tableName, + ULng32 tableNameLen, + char * tableLocation, + char * partnLocation, + char * hostName, + Lng32 portNum, + Int64 modTS, + ex_cri_desc * given_cri_desc, + ex_cri_desc * returned_cri_desc, + queue_index down, + queue_index up, + Lng32 num_buffers, + ULng32 buffer_size) + : ComTdbExeUtil(ComTdbExeUtil::HIVE_TRUNCATE_, + NULL, 0, (Int16)SQLCHARSETCODE_UNKNOWN, + tableName, tableNameLen, + NULL, 0, + NULL, 0, + NULL, + NULL, 0, + given_cri_desc, returned_cri_desc, + down, up, + num_buffers, buffer_size), + flags_(0), + tableLocation_(tableLocation), + partnLocation_(partnLocation), + hdfsHost_(hostName), + hdfsPort_(portNum), + modTS_(modTS) +{ + setNodeType(ComTdb::ex_HIVE_TRUNCATE); +} + +Long ComTdbExeUtilHiveTruncate::pack(void * space) +{ + if (tableLocation_) + tableLocation_.pack(space); + + if (hdfsHost_) + hdfsHost_.pack(space); + + if (partnLocation_) + partnLocation_.pack(space); + + return ComTdbExeUtil::pack(space); +} + +Lng32 ComTdbExeUtilHiveTruncate::unpack(void * base, void * reallocator) +{ + if(tableLocation_.unpack(base)) + return -1; + + if(hdfsHost_.unpack(base)) + return -1; + + if (partnLocation_.unpack(base)) + return -1; + + return ComTdbExeUtil::unpack(base, reallocator); +} + +void ComTdbExeUtilHiveTruncate::displayContents(Space * space, + ULng32 flag) +{ + ComTdb::displayContents(space,flag & 0xFFFFFFFE); + + if(flag & 0x00000008) + { + char buf[500]; + str_sprintf(buf, "\nFor ComTdbExeUtilHiveTruncate :"); + space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(short)); + + if (getTableName() != NULL) + { + str_sprintf(buf,"Tablename = %s ",getTableName()); + space->allocateAndCopyToAlignedSpace(buf, str_len(buf), + sizeof(short)); + } + + if (getTableLocation() != NULL) + { + str_sprintf(buf,"tableLocation_ = %s ", getTableLocation()); + space->allocateAndCopyToAlignedSpace(buf, str_len(buf), + sizeof(short)); + } + + if (getPartnLocation() != NULL) + { + str_sprintf(buf,"partnLocation_ = %s ", getPartnLocation()); + space->allocateAndCopyToAlignedSpace(buf, str_len(buf), + sizeof(short)); + } + + } + + if (flag & 0x00000001) + { + displayExpression(space,flag); + displayChildren(space,flag); + } + +} + +/////////////////////////////////////////////////////////////////////////// +// // Methods for class ComTdbExeUtilGetStatistics // /////////////////////////////////////////////////////////////////////////// http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/comexe/ComTdbExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbExeUtil.h b/core/sql/comexe/ComTdbExeUtil.h index 9c4990e..8d40556 100644 --- a/core/sql/comexe/ComTdbExeUtil.h +++ b/core/sql/comexe/ComTdbExeUtil.h @@ -89,7 +89,8 @@ public: HBASE_LOAD_ = 32, HBASE_UNLOAD_ = 33, HBASE_UNLOAD_TASK_ = 34, - GET_QID_ = 35 + GET_QID_ = 35, + HIVE_TRUNCATE_ = 36 }; ComTdbExeUtil() @@ -1514,12 +1515,7 @@ public: queue_index down, queue_index up, Lng32 num_buffers, - ULng32 buffer_size, - NABoolean ishiveTruncate = FALSE, - char * hiveTableLocation = NULL, - char * hiveHostName = NULL, - Lng32 hivePortNum = 0, - Int64 hiveModTS = -1 + ULng32 buffer_size ); Long pack(void *); @@ -1533,10 +1529,7 @@ public: virtual const char *getNodeName() const { - if (isHiveTruncate()) - return "HIVE_TRUNCATE"; - else - return "FAST_DELETE"; + return "FAST_DELETE"; }; Queue* getIndexList() { return indexList_; } @@ -1555,26 +1548,6 @@ public: Int64 getObjectUID() { return objectUID_; } - char * getHiveTableLocation() const - { - return hiveTableLocation_; - } - - char * getHiveHdfsHost() const - { - return hiveHdfsHost_; - } - - Lng32 getHiveHdfsPort() const - { - return hiveHdfsPort_; - } - - Lng32 getHiveModTS() const - { - return hiveModTS_; - } - // --------------------------------------------------------------------- // Used by the internal SHOWPLAN command to get attributes of a TDB. // --------------------------------------------------------------------- @@ -1604,10 +1577,6 @@ public: {(v ? flags_ |= OFFLINE_TABLE : flags_ &= ~OFFLINE_TABLE); }; NABoolean offlineTable() { return (flags_ & OFFLINE_TABLE) != 0; }; - void setIsHiveTruncate(NABoolean v) - {(v ? flags_ |= IS_HIVE_TRUNCATE : flags_ &= ~IS_HIVE_TRUNCATE); }; - NABoolean isHiveTruncate() const { return (flags_ & IS_HIVE_TRUNCATE) != 0; }; - void setDoLabelPurgedata(NABoolean v) {(v ? flags_ |= DO_LABEL_PURGEDATA: flags_ &= ~DO_LABEL_PURGEDATA); }; NABoolean doLabelPurgedata() { return (flags_ & DO_LABEL_PURGEDATA) != 0; }; @@ -1621,8 +1590,7 @@ private: DO_PARALLEL_DELETE = 0x0008, DO_PARALLEL_DELETE_IF_XN = 0x0010, OFFLINE_TABLE = 0x0020, - DO_LABEL_PURGEDATA = 0x0040, - IS_HIVE_TRUNCATE = 0x0080 + DO_LABEL_PURGEDATA = 0x0040 }; // list of indexes on the table. @@ -1647,13 +1615,80 @@ private: NABasicPtr lobNumArray_; // 40-47 Int64 objectUID_; // 48-55 - // hiveTable loaction will be extended for partitions later - NABasicPtr hiveTableLocation_; // 56-63 - NABasicPtr hiveHdfsHost_; // 64-71 - Int32 hiveHdfsPort_; // 72-75 - char fillers1_[4]; // 76-79 - Int64 hiveModTS_; // 80-87 - char fillersComTdbExeUtilFastDelete_[40]; // 88-127 +}; + +class ComTdbExeUtilHiveTruncate : public ComTdbExeUtil +{ +public: + ComTdbExeUtilHiveTruncate() + : ComTdbExeUtil() + {} + + ComTdbExeUtilHiveTruncate(char * tableName, + ULng32 tableNameLen, + char * tableLocation, + char * partnLocation, + char * hostName, + Lng32 portNum, + Int64 modTS, + ex_cri_desc * given_cri_desc, + ex_cri_desc * returned_cri_desc, + queue_index down, + queue_index up, + Lng32 num_buffers, + ULng32 buffer_size + ); + + Long pack(void *); + Lng32 unpack(void *, void * reallocator); + + // --------------------------------------------------------------------- + // Redefine virtual functions required for Versioning. + //---------------------------------------------------------------------- + virtual short getClassSize() {return (short)sizeof(ComTdbExeUtilHiveTruncate);} + + virtual const char *getNodeName() const + { + return "HIVE_TRUNCATE"; + }; + + char * getTableLocation() const + { + return tableLocation_; + } + + char * getHdfsHost() const + { + return hdfsHost_; + } + + Lng32 getHdfsPort() const + { + return hdfsPort_; + } + + Lng32 getModTS() const + { + return modTS_; + } + + char * getPartnLocation() const + { + return partnLocation_; + } + + // --------------------------------------------------------------------- + // Used by the internal SHOWPLAN command to get attributes of a TDB. + // --------------------------------------------------------------------- + NA_EIDPROC void displayContents(Space *space, ULng32 flag); + +private: + NABasicPtr tableLocation_; // 00-07 + NABasicPtr partnLocation_; // 08-15 + NABasicPtr hdfsHost_; // 16-23 + Int64 modTS_; // 24-31 + Int32 hdfsPort_; // 32-35 + UInt32 flags_; // 36-39 }; class ComTdbExeUtilGetStatistics : public ComTdbExeUtil http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/executor/ExComTdb.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExComTdb.cpp b/core/sql/executor/ExComTdb.cpp index ad17627..d6c0145 100644 --- a/core/sql/executor/ExComTdb.cpp +++ b/core/sql/executor/ExComTdb.cpp @@ -420,7 +420,6 @@ NA_EIDPROC char *ComTdb::findVTblPtrExe(short classID) break; } - case ex_FAST_DELETE: { #pragma nowarn(1506) // warning elimination @@ -429,6 +428,14 @@ NA_EIDPROC char *ComTdb::findVTblPtrExe(short classID) break; } + case ex_HIVE_TRUNCATE: + { +#pragma nowarn(1506) // warning elimination + GetVTblPtr(vtblptr,ExExeUtilHiveTruncateTdb); +#pragma warn(1506) // warning elimination + break; + } + case ex_PROCESS_VOLATILE_TABLE: { #pragma nowarn(1506) // warning elimination http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/executor/ExExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h index c52eb44..f84ed7c 100755 --- a/core/sql/executor/ExExeUtil.h +++ b/core/sql/executor/ExExeUtil.h @@ -86,6 +86,7 @@ class ExExeUtilTdb; class ExExeUtilDisplayExplainTdb; class ExExeUtilDisplayExplainComplexTdb; class ExExeUtilFastDeleteTdb; +class ExExeUtilHiveTruncateTdb; class ExExeUtilSuspendTdb; class ExExeUtilSuspendTcb; class ExpHbaseInterface; @@ -429,6 +430,7 @@ class ExExeUtilPrivateState : public ex_tcb_private_state friend class ExExeUtilCleanupVolatileTablesTcb; friend class ExExeUtilCreateTableAsTcb; friend class ExExeUtilFastDeleteTcb; + friend class ExExeUtilHiveTruncateTcb; friend class ExExeUtilAQRTcb; friend class ExExeUtilHBaseBulkLoadTcb; friend class ExExeUtilHBaseBulkUnLoadTcb; @@ -3276,6 +3278,57 @@ protected: }; +// ----------------------------------------------------------------------- +// ExExeUtilHiveTruncateTdb +// ----------------------------------------------------------------------- +class ExExeUtilHiveTruncateTdb : public ComTdbExeUtilHiveTruncate +{ + public: + + // --------------------------------------------------------------------- + // Constructor is only called to instantiate an object used for + // retrieval of the virtual table function pointer of the class while + // unpacking. An empty constructor is enough. + // --------------------------------------------------------------------- + NA_EIDPROC ExExeUtilHiveTruncateTdb() + {} + + NA_EIDPROC virtual ~ExExeUtilHiveTruncateTdb() + {} + + // --------------------------------------------------------------------- + // Build a TCB for this TDB. Redefined in the Executor project. + // --------------------------------------------------------------------- + NA_EIDPROC virtual ex_tcb *build(ex_globals *globals); + + private: + // --------------------------------------------------------------------- + // !!!!!!! IMPORTANT -- NO DATA MEMBERS ALLOWED IN EXECUTOR TDB !!!!!!!! + // ********************************************************************* + // The Executor TDB's are only used for the sole purpose of providing a + // way to supplement the Compiler TDB's (in comexe) with methods whose + // implementation depends on Executor objects. This is done so as to + // decouple the Compiler from linking in Executor objects unnecessarily. + // + // When a Compiler generated TDB arrives at the Executor, the same data + // image is "cast" as an Executor TDB after unpacking. Therefore, it is + // a requirement that a Compiler TDB has the same object layout as its + // corresponding Executor TDB. As a result of this, all Executor TDB's + // must have absolutely NO data members, but only member functions. So, + // if you reach here with an intention to add data members to a TDB, ask + // yourself two questions: + // + // 1. Are those data members Compiler-generated? + // If yes, put them in the ComTdbDLL instead. + // If no, they should probably belong to someplace else (like TCB). + // + // 2. Are the classes those data members belong defined in the executor + // project? + // If your answer to both questions is yes, you might need to move + // the classes to the comexe project. + // --------------------------------------------------------------------- +}; + /////////////////////////////////////////////////////////////// // ExExeUtilHiveTruncateTcb /////////////////////////////////////////////////////////////// @@ -3283,8 +3336,8 @@ class ExExeUtilHiveTruncateTcb : public ExExeUtilTcb { public: // Constructor - ExExeUtilHiveTruncateTcb(const ComTdbExeUtilFastDelete & exe_util_tdb, - ex_globals * glob = 0); + ExExeUtilHiveTruncateTcb(const ComTdbExeUtilHiveTruncate & exe_util_tdb, + ex_globals * glob = 0); ~ExExeUtilHiveTruncateTcb(); @@ -3304,8 +3357,8 @@ class ExExeUtilHiveTruncateTcb : public ExExeUtilTcb DONE_ }; - ExExeUtilFastDeleteTdb & fdTdb() const - {return (ExExeUtilFastDeleteTdb &) tdb;}; + ExExeUtilHiveTruncateTdb & htTdb() const + {return (ExExeUtilHiveTruncateTdb &) tdb;}; short injectError(const char * val); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/executor/ExExeUtilMisc.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtilMisc.cpp b/core/sql/executor/ExExeUtilMisc.cpp index 9c86f72..ea8a809 100644 --- a/core/sql/executor/ExExeUtilMisc.cpp +++ b/core/sql/executor/ExExeUtilMisc.cpp @@ -69,15 +69,7 @@ ex_tcb * ExExeUtilFastDeleteTdb::build(ex_globals * glob) { ExExeUtilTcb * exe_util_tcb; - - if (!isHiveTruncate()) - { - exe_util_tcb = new(glob->getSpace()) ExExeUtilFastDeleteTcb(*this, glob); - } - else - { - exe_util_tcb = new(glob->getSpace()) ExExeUtilHiveTruncateTcb(*this, glob); - } + exe_util_tcb = new(glob->getSpace()) ExExeUtilFastDeleteTcb(*this, glob); exe_util_tcb->registerSubtasks(); return (exe_util_tcb); @@ -2229,10 +2221,24 @@ ExExeUtilPopulateInMemStatsPrivateState::~ExExeUtilPopulateInMemStatsPrivateStat //////////////////////////////////////////////////////////////// +// Constructor for class ExExeUtilHiveTruncateTdb +/////////////////////////////////////////////////////////////// +ex_tcb * ExExeUtilHiveTruncateTdb::build(ex_globals * glob) +{ + ExExeUtilTcb * exe_util_tcb; + + exe_util_tcb = new(glob->getSpace()) ExExeUtilHiveTruncateTcb(*this, glob); + exe_util_tcb->registerSubtasks(); + + return (exe_util_tcb); +} + + +//////////////////////////////////////////////////////////////// // Constructor for class ExExeUtilHiveTruncateTcb /////////////////////////////////////////////////////////////// ExExeUtilHiveTruncateTcb::ExExeUtilHiveTruncateTcb( - const ComTdbExeUtilFastDelete & exe_util_tdb, + const ComTdbExeUtilHiveTruncate & exe_util_tdb, ex_globals * glob) : ExExeUtilTcb( exe_util_tdb, NULL, glob) { @@ -2284,7 +2290,7 @@ short ExExeUtilHiveTruncateTcb::work() case INITIAL_: { - if (fdTdb().getHiveModTS() > 0) + if (htTdb().getModTS() > 0) step_ = DATA_MOD_CHECK_; else step_ = EMPTY_DIRECTORY_; @@ -2295,10 +2301,12 @@ short ExExeUtilHiveTruncateTcb::work() { cliRC = ExpLOBinterfaceDataModCheck (lobGlob_, - fdTdb().getHiveTableLocation(), - fdTdb().getHiveHdfsHost(), - fdTdb().getHiveHdfsPort(), - fdTdb().getHiveModTS(), + (htTdb().getPartnLocation() ? + htTdb().getPartnLocation() : + htTdb().getTableLocation()), + htTdb().getHdfsHost(), + htTdb().getHdfsPort(), + htTdb().getModTS(), 0); if (cliRC < 0) @@ -2340,10 +2348,12 @@ short ExExeUtilHiveTruncateTcb::work() cliRC = ExpLOBinterfaceEmptyDirectory( lobGlob_, (char*)"", //name is empty - fdTdb().getHiveTableLocation(), + (htTdb().getPartnLocation() ? + htTdb().getPartnLocation() : + htTdb().getTableLocation()), Lob_HDFS_File, - fdTdb().getHiveHdfsHost(), - fdTdb().getHiveHdfsPort(), + htTdb().getHdfsHost(), + htTdb().getHdfsPort(), 0 , 1 , 0); @@ -2361,6 +2371,28 @@ short ExExeUtilHiveTruncateTcb::work() "HDFS", (char*)"ExpLOBInterfaceEmptyDirectory", getLobErrStr(intParam1)); + + char reason[200]; + + strcpy(reason, " "); + if (intParam1 == LOB_DIR_NAME_ERROR) + { + if (htTdb().getPartnLocation()) + strcpy(reason, "Reason: specified partition does not exist"); + else + strcpy(reason, "Reason: specified table location does not exist"); + } + else if (intParam1 == LOB_DATA_FILE_DELETE_ERROR) + { + strcpy(reason, "Reason: error occurred during deletion of one or more files at the specified location"); + } + + ExRaiseSqlError(getHeap(), &diagsArea, + (ExeErrorCode)(EXE_HIVE_TRUNCATE_ERROR), NULL, + NULL, NULL, NULL, + reason, + NULL, NULL); + pentry_down->setDiagsArea(diagsArea); step_ = ERROR_; } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/exp/ExpErrorEnums.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpErrorEnums.h b/core/sql/exp/ExpErrorEnums.h index 987fe7e..bed50de 100644 --- a/core/sql/exp/ExpErrorEnums.h +++ b/core/sql/exp/ExpErrorEnums.h @@ -75,9 +75,11 @@ enum ExeErrorCode EXE_CANCEL_NOT_AUTHORIZED = 8029, EXE_CANCEL_NOT_POSSIBLE = 8031, - EXE_NO_QID_EXPLAIN_INFO = 8032, + EXE_NO_QID_EXPLAIN_INFO = 8032, EXE_EXPLAIN_PLAN_TOO_LARGE = 8033, + EXE_HIVE_TRUNCATE_ERROR = 8034, + // --------------------------------------------------------------------- // Data integrity errors // --------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/exp/ExpLOBaccess.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp index 46f5d1a..9446327 100644 --- a/core/sql/exp/ExpLOBaccess.cpp +++ b/core/sql/exp/ExpLOBaccess.cpp @@ -464,43 +464,73 @@ Ex_Lob_Error ExLob::dataModCheck( return LOB_OPER_OK; } -Ex_Lob_Error ExLob::emptyDirectory() +Ex_Lob_Error ExLob::emptyDirectory(char *dirPath, + ExLobGlobals *lobGlobals) { - Ex_Lob_Error err; + int retcode = 0; - int numExistingFiles=0; - hdfsFileInfo *fileInfos = hdfsGetPathInfo(fs_, lobDataFile_); - if (fileInfos == NULL) + hdfsFileInfo *fileInfos = hdfsGetPathInfo(fs_, dirPath); + if (fileInfos == NULL) { - return LOB_DIR_NAME_ERROR; + hdfsDisconnect(fs_); + fs_ = hdfsConnect(hdfsServer_, hdfsPort_); + if (fs_ == NULL) + return LOB_HDFS_CONNECT_ERROR; + + fileInfos = hdfsGetPathInfo(fs_, lobDataFile_); + if (fileInfos == NULL) + return LOB_DIR_NAME_ERROR; + + if (lobGlobals) + lobGlobals->setHdfsFs(fs_); + } + + Lng32 currNumFilesInDir = 0; + fileInfos = hdfsListDirectory(fs_, dirPath, &currNumFilesInDir); + if ((currNumFilesInDir > 0) && (fileInfos == NULL)) + { + return LOB_DATA_FILE_NOT_FOUND_ERROR; } - fileInfos = hdfsListDirectory(fs_, lobDataFile_, &numExistingFiles); - if (fileInfos == NULL) // empty directory + if ((currNumFilesInDir == 0) && (fileInfos == NULL)) // empty directory { return LOB_OPER_OK; } - NABoolean error = FALSE; - for (int i = 0; ((NOT error) && (i < numExistingFiles)); i++) + // delete all files in this directory + NABoolean error = FALSE; + for (Lng32 i = 0; i < currNumFilesInDir; i++) { - // if dir, recursively delete it and everything under it - int retCode = hdfsDelete(fs_, fileInfos[i].mName, 1); - if (retCode !=0) - { - error = TRUE; - } - } + hdfsFileInfo &fileInfo = fileInfos[i]; + if (fileInfo.mKind == kObjectKindFile) + { + retcode = hdfsDelete(fs_, fileInfo.mName, 0); + if (retcode != 0) + error = TRUE; + } + } // for - if (fileInfos) + // recursively delete all files in sub-dirs + for (Lng32 i = 0; i < currNumFilesInDir; i++) { - hdfsFreeFileInfo(fileInfos, numExistingFiles); - } + hdfsFileInfo &fileInfo = fileInfos[i]; + if (fileInfo.mKind == kObjectKindDirectory) + { + retcode = emptyDirectory(fileInfo.mName, lobGlobals); + if (retcode != LOB_OPER_OK) + error = TRUE; + } + } // for - if (error) - return LOB_DATA_FILE_DELETE_ERROR; + if (fileInfos) + { + hdfsFreeFileInfo(fileInfos, currNumFilesInDir); + } - return LOB_OPER_OK; + if (error) + return LOB_DATA_FILE_DELETE_ERROR; + + return LOB_OPER_OK; } struct MemoryStruct { @@ -2473,8 +2503,9 @@ Ex_Lob_Error ExLobsOper ( case Lob_Empty_Directory: lobPtr->initialize(fileName, EX_LOB_RW, - dir, storage, hdfsServer, hdfsPort, dir, bufferSize, replication, blockSize); - err = lobPtr->emptyDirectory(); + dir, storage, hdfsServer, hdfsPort, dir, bufferSize, + replication, blockSize); + err = lobPtr->emptyDirectory(dir, lobGlobals); break; case Lob_Data_Mod_Check: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/exp/ExpLOBaccess.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h index 2cce09e..176e400 100644 --- a/core/sql/exp/ExpLOBaccess.h +++ b/core/sql/exp/ExpLOBaccess.h @@ -494,7 +494,8 @@ class ExLob Int64 modTS, Lng32 numOfPartLevels); - Ex_Lob_Error emptyDirectory(); + Ex_Lob_Error emptyDirectory(char* dirPath, ExLobGlobals* lobGlobals); + ExLobStats *getStats() { return &stats_; } NAHeap *getLobGlobalHeap() { return lobGlobalHeap_;} ExLobRequest *getRequest() { return &request_; } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/generator/GenPreCode.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenPreCode.cpp b/core/sql/generator/GenPreCode.cpp index 56a1b95..dc7fb0e 100644 --- a/core/sql/generator/GenPreCode.cpp +++ b/core/sql/generator/GenPreCode.cpp @@ -5687,6 +5687,16 @@ RelExpr * ExeUtilFastDelete::preCodeGen(Generator * generator, return ExeUtilExpr::preCodeGen(generator,externalInputs,pulledNewInputs); } +RelExpr * ExeUtilHiveTruncate::preCodeGen(Generator * generator, + const ValueIdSet & externalInputs, + ValueIdSet &pulledNewInputs) +{ + if (nodeIsPreCodeGenned()) + return this; + + return ExeUtilExpr::preCodeGen(generator,externalInputs,pulledNewInputs); +} + RelExpr * ExeUtilLobExtract::preCodeGen(Generator * generator, const ValueIdSet & externalInputs, ValueIdSet &pulledNewInputs) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/generator/GenRelExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelExeUtil.cpp b/core/sql/generator/GenRelExeUtil.cpp index 62b96c8..cf72a7f 100644 --- a/core/sql/generator/GenRelExeUtil.cpp +++ b/core/sql/generator/GenRelExeUtil.cpp @@ -3209,17 +3209,7 @@ short ExeUtilFastDelete::codeGen(Generator * generator) } } - - char * hiveTableLocation = NULL; - char * hiveHdfsHost = NULL; - Int32 hiveHdfsPort = getHiveHdfsPort(); - - hiveTableLocation = - space->AllocateAndCopyToAlignedSpace (getHiveTableLocation(), 0); - hiveHdfsHost = - space->AllocateAndCopyToAlignedSpace (getHiveHostName(), 0); - - Lng32 numEsps = -1; + Lng32 numEsps = -1; ComTdbExeUtilFastDelete * exe_util_tdb = new(space) ComTdbExeUtilFastDelete(tablename, strlen(tablename), @@ -3238,10 +3228,7 @@ short ExeUtilFastDelete::codeGen(Generator * generator) (queue_index)getDefault(GEN_DDL_SIZE_DOWN), (queue_index)getDefault(GEN_DDL_SIZE_UP), getDefault(GEN_DDL_NUM_BUFFERS), - getDefault(GEN_DDL_BUFFER_SIZE), - isHiveTable(), - hiveTableLocation, hiveHdfsHost, hiveHdfsPort, - hiveModTS_); + getDefault(GEN_DDL_BUFFER_SIZE)); if (doPurgedataCat_) exe_util_tdb->setDoPurgedataCat(TRUE); @@ -3258,9 +3245,6 @@ short ExeUtilFastDelete::codeGen(Generator * generator) if (doLabelPurgedata_) exe_util_tdb->setDoLabelPurgedata(TRUE); - if (CmpCommon::getDefault(EXE_PARALLEL_PURGEDATA_WARNINGS) == DF_ON) - exe_util_tdb->setReturnPurgedataWarn(TRUE); - if ((getUtilTableDesc()) && (getUtilTableDesc()->getNATable()) && (getUtilTableDesc()->getNATable()->isAnMV())) @@ -3287,6 +3271,98 @@ short ExeUtilFastDelete::codeGen(Generator * generator) return 0; } +///////////////////////////////////////////////////////// +// +// ExeUtilHiveTruncate::codeGen() +// +///////////////////////////////////////////////////////// +short ExeUtilHiveTruncate::codeGen(Generator * generator) +{ + ExpGenerator * expGen = generator->getExpGenerator(); + Space * space = generator->getSpace(); + + // allocate a map table for the retrieved columns + generator->appendAtEnd(); + + ex_cri_desc * givenDesc + = generator->getCriDesc(Generator::DOWN); + + ex_cri_desc * returnedDesc + = new(space) ex_cri_desc(givenDesc->noTuples() + 1, space); + + ex_cri_desc * workCriDesc = new(space) ex_cri_desc(4, space); + const Int32 work_atp = 1; + const Int32 exe_util_row_atp_index = 2; + + short rc = processOutputRow(generator, work_atp, exe_util_row_atp_index, + returnedDesc); + if (rc) + { + return -1; + } + + char * partn_loc = NULL; + if (pl_) + { + NAString partnLoc = getHiveTableLocation(); + partnLoc += "/"; + + for (Lng32 i = 0; i < pl_->entries(); i++) + { + const NAString *cs = (*pl_)[i]; + + partnLoc += *cs; + partnLoc += "/"; + + } + + partn_loc = + space->allocateAndCopyToAlignedSpace + (partnLoc.data(), partnLoc.length(), 0); + } + + char * tablename = NULL; + tablename = space->AllocateAndCopyToAlignedSpace + (generator->genGetNameAsAnsiNAString(getTableName()), 0); + + char * hiveTableLocation = NULL; + char * hiveHdfsHost = NULL; + Int32 hiveHdfsPort = getHiveHdfsPort(); + + hiveTableLocation = + space->AllocateAndCopyToAlignedSpace (getHiveTableLocation(), 0); + hiveHdfsHost = + space->AllocateAndCopyToAlignedSpace (getHiveHostName(), 0); + + ComTdbExeUtilHiveTruncate * exe_util_tdb = new(space) + ComTdbExeUtilHiveTruncate(tablename, strlen(tablename), + hiveTableLocation, partn_loc, + hiveHdfsHost, hiveHdfsPort, + hiveModTS_, + (ex_cri_desc *)(generator->getCriDesc(Generator::DOWN)), + (ex_cri_desc *)(generator->getCriDesc(Generator::DOWN)), + (queue_index)getDefault(GEN_DDL_SIZE_DOWN), + (queue_index)getDefault(GEN_DDL_SIZE_UP), + getDefault(GEN_DDL_NUM_BUFFERS), + getDefault(GEN_DDL_BUFFER_SIZE)); + + generator->initTdbFields(exe_util_tdb); + + if(!generator->explainDisabled()) { + generator->setExplainTuple( + addExplainInfo(exe_util_tdb, 0, 0, generator)); + } + + // no tupps are returned + generator->setCriDesc((ex_cri_desc *)(generator->getCriDesc(Generator::DOWN)), + Generator::UP); + generator->setGenObj(this, exe_util_tdb); + + generator->setTransactionFlag(0); // transaction is not needed. + + return 0; +} + //////////////////////////////////////////////////////////////////// // class ExeUtilRegionStats //////////////////////////////////////////////////////////////////// http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/optimizer/BindRelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp index 8188b57..45ac242 100644 --- a/core/sql/optimizer/BindRelExpr.cpp +++ b/core/sql/optimizer/BindRelExpr.cpp @@ -9183,20 +9183,7 @@ RelExpr *Insert::bindNode(BindWA *bindWA) if (getOverwriteHiveTable()) { RelExpr * newRelExpr = new (bindWA->wHeap()) - ExeUtilFastDelete(getTableName(), - NULL, - (char*)"hive_truncate", - CharInfo::ISO88591, - FALSE, - TRUE, - TRUE, - TRUE, - bindWA->wHeap(), - TRUE, - new (bindWA->wHeap()) NAString(tableDir), - new (bindWA->wHeap()) NAString(hostName), - hdfsPort, - hTabStats->getModificationTS()); + ExeUtilHiveTruncate(getTableName(), NULL, bindWA->wHeap()); //new root to prevent error 4056 when binding newRelExpr = new (bindWA->wHeap()) RelRoot(newRelExpr); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/optimizer/OptPhysRelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/OptPhysRelExpr.cpp b/core/sql/optimizer/OptPhysRelExpr.cpp index 35229aa..1acfd76 100644 --- a/core/sql/optimizer/OptPhysRelExpr.cpp +++ b/core/sql/optimizer/OptPhysRelExpr.cpp @@ -12372,37 +12372,8 @@ Context* RelRoot::createContextForAChild(Context* myContext, // final adjustment to countOfCPUs and pipelinesPerCPU - special cases // - // Check CQD EXE_PARALLEL_PURGEDATA and do the #esp = #partition parallel - // plan when the CQD is not set to OFF. See ExeUtilFastDelete::bindNode() - // on legal values for the CQD. + RelExpr* x = child(0).getGroupAttr()->getLogExprForSynthesis(); - RelExpr* x = child(0).getGroupAttr()->getLogExprForSynthesis(); - - if (CmpCommon::getDefault(EXE_PARALLEL_PURGEDATA) != DF_OFF) - { - - if ( x && x->getOperatorType() == REL_UNARY_DELETE && - ((Delete*)x)->isFastDelete() // fast delete is turned on - // for DELETE USING PURGEDATA FROM <t> - ) - { - PartitioningFunction *pf = ((Delete*)x)->getScanIndexDesc() - ->getPartitioningFunction(); - - const NodeMap* np; - if ( pf && (np = pf->getNodeMap()) && np->getNumEntries() > 1 ) { - // set countOfCPUs to the number of partitions - UInt32 partns = np->getNumEntries(); - countOfCPUs = partns; - pipelinesPerCPU = 1; - CURRSTMT_OPTDEFAULTS->setRequiredESPs(partns); - CURRSTMT_OPTDEFAULTS->setRequiredScanDescForFastDelete( - ((Delete*)x)->getScanIndexDesc()); - canAdjustDoP = FALSE; - } - } - } - // for multi-commit DELETE (expressed as DELETE WITH MULTI COMMIT FROM <t>) if ( containsLRU() ) { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/optimizer/RelExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExeUtil.cpp b/core/sql/optimizer/RelExeUtil.cpp index 0b439ae..5830826 100644 --- a/core/sql/optimizer/RelExeUtil.cpp +++ b/core/sql/optimizer/RelExeUtil.cpp @@ -354,16 +354,11 @@ const NAString ExeUtilExpr::getText() const break; case FAST_DELETE_: - { - if (((ExeUtilFastDelete*)this)->isHiveTable()) - { - result = "HIVE_TRUNCATE"; - } - else - { - result = "FAST_DELETE"; - } - } + result = "FAST_DELETE"; + break; + + case HIVE_TRUNCATE_: + result = "HIVE_TRUNCATE"; break; case GET_STATISTICS_: @@ -822,11 +817,7 @@ RelExpr * ExeUtilFastDelete::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap noLog_, ignoreTrigger_, isPurgedata_, - outHeap, - isHiveTable_, - &hiveTableLocation_, - &hiveHostName_, - hiveHdfsPort_); + outHeap); else result = (ExeUtilFastDelete *) derivedNode; @@ -837,12 +828,28 @@ RelExpr * ExeUtilFastDelete::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap result->numLOBs_ = numLOBs_; result->lobNumArray_ = lobNumArray_; - result->isHiveTable_ = isHiveTable_; + + return ExeUtilExpr::copyTopNode(result, outHeap); +} + +// ----------------------------------------------------------------------- +// Member functions for class ExeUtilHiveTruncate +// ----------------------------------------------------------------------- +RelExpr * ExeUtilHiveTruncate::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap) +{ + ExeUtilHiveTruncate *result; + + if (derivedNode == NULL) + result = new (outHeap) ExeUtilHiveTruncate(getTableName(), + pl_, + outHeap); + else + result = (ExeUtilHiveTruncate *) derivedNode; + result->hiveTableLocation_= hiveTableLocation_; result->hiveHostName_ = hiveHostName_; result->hiveHdfsPort_ = hiveHdfsPort_; - return ExeUtilExpr::copyTopNode(result, outHeap); } @@ -4975,88 +4982,100 @@ RelExpr * ExeUtilFastDelete::bindNode(BindWA *bindWA) if (bindWA->errStatus()) return this; - NATable *naTable = NULL; - - if (NOT doPurgedataCat_) + // do not do override schema for this + bindWA->setToOverrideSchema(FALSE); + + NATable * naTable = bindWA->getNATable(getTableName()); + if ((!naTable) || + (bindWA->errStatus())) + return this; + + if ((getTableName().isHive()) || + (naTable->isHiveTable())) { - // do not do override schema for this - bindWA->setToOverrideSchema(FALSE); - - naTable = bindWA->getNATable(getTableName()); - if (getTableName().isHive()) - { - if (! naTable) - { - *CmpCommon::diags() << DgSqlCode(-4222) << DgString0("PURGEDATA"); - bindWA->setErrStatus(); - return NULL; - } + *CmpCommon::diags() << DgSqlCode(-3242) + << DgString0("Purgedata is not allowed for hive tables. Use Truncate command."); + bindWA->setErrStatus(); + return NULL; + } + + if (! getTableName().isSeabase()) + { + *CmpCommon::diags() << DgSqlCode(-4222) << DgString0("PURGEDATA"); + bindWA->setErrStatus(); + return NULL; + } + + DDLExpr * ddlExpr = new(bindWA->wHeap()) DDLExpr(TRUE, + getTableName(), + getStmtText(), + CharInfo::UnknownCharSet); + RelExpr * boundExpr = ddlExpr->bindNode(bindWA); - const HHDFSTableStats* hTabStats = - naTable->getClusteringIndex()->getHHDFSTableStats(); - - isHiveTable_ = TRUE; - - const char * hiveTablePath = (*hTabStats)[0]->getDirName(); - NAString hostName; - Int32 hdfsPort; - NAString tableDir; - - NABoolean result = ((HHDFSTableStats* )hTabStats)->splitLocation - (hiveTablePath, hostName, hdfsPort, tableDir) ; - if (!result) - { - *CmpCommon::diags() << DgSqlCode(-4224) - << DgString0(hiveTablePath); - bindWA->setErrStatus(); - return this; - } - - hiveTableLocation_ = tableDir; - hiveHostName_ = hostName; - hiveHdfsPort_ = hdfsPort; - hiveModTS_ = -1; - } - else if (getTableName().isSeabase()) - { - if (bindWA->errStatus()) - return this; - - DDLExpr * ddlExpr = new(bindWA->wHeap()) DDLExpr(TRUE, - getTableName(), - getStmtText(), - CharInfo::UnknownCharSet); - RelExpr * boundExpr = ddlExpr->bindNode(bindWA); - return boundExpr; - } - - if (bindWA->errStatus()) - { - naTable = NULL; - CmpCommon::diags()->clear(); - bindWA->resetErrStatus(); - } + return boundExpr; +} + +// ----------------------------------------------------------------------- +// member functions for class ExeUtilHiveTruncate +// ----------------------------------------------------------------------- +RelExpr * ExeUtilHiveTruncate::bindNode(BindWA *bindWA) +{ + if (nodeIsBound()) + { + bindWA->getCurrentScope()->setRETDesc(getRETDesc()); + return this; } - RelExpr * boundExpr = ExeUtilExpr::bindNode(bindWA); - if (bindWA->errStatus()) - return NULL; + bindChildren(bindWA); + if (bindWA->errStatus()) + return this; - if ((! getTableName().isSeabase()) && - (! getTableName().isHbase()) && - (! getTableName().isHive())) + NATable *naTable = NULL; + + // do not do override schema for this + bindWA->setToOverrideSchema(FALSE); + + naTable = bindWA->getNATable(getTableName()); + if ((!naTable) || + (bindWA->errStatus())) + return this; + + if ((NOT getTableName().isHive()) || + (!naTable->isHiveTable())) { - *CmpCommon::diags() << DgSqlCode(-4222) << DgString0("PURGEDATA"); + *CmpCommon::diags() << DgSqlCode(-3242) + << DgString0("Truncate is only allowed for hive tables."); bindWA->setErrStatus(); return NULL; } - if (naTable && (!naTable->isHiveTable())) + const HHDFSTableStats* hTabStats = + naTable->getClusteringIndex()->getHHDFSTableStats(); + + const char * hiveTablePath = (*hTabStats)[0]->getDirName(); + NAString hostName; + Int32 hdfsPort; + NAString tableDir; + + NABoolean result = ((HHDFSTableStats* )hTabStats)->splitLocation + (hiveTablePath, hostName, hdfsPort, tableDir) ; + if (!result) { - *CmpCommon::diags() << DgSqlCode(-4222) << DgString0("PURGEDATA"); + *CmpCommon::diags() << DgSqlCode(-4224) + << DgString0(hiveTablePath); bindWA->setErrStatus(); - return NULL; + return this; } + + hiveTableLocation_ = tableDir; + hiveHostName_ = hostName; + hiveHdfsPort_ = hdfsPort; + hiveModTS_ = -1; + + RelExpr * boundExpr = ExeUtilExpr::bindNode(bindWA); + if (bindWA->errStatus()) + return NULL; + return boundExpr; } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/optimizer/RelExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExeUtil.h b/core/sql/optimizer/RelExeUtil.h index 5c782e9..b97cf72 100644 --- a/core/sql/optimizer/RelExeUtil.h +++ b/core/sql/optimizer/RelExeUtil.h @@ -563,7 +563,8 @@ public: HBASE_UNLOAD_ = 35, HBASE_UNLOAD_TASK_ = 36, ORC_FAST_AGGR_ = 37, - GET_QID_ = 38 + GET_QID_ = 38, + HIVE_TRUNCATE_ = 39 }; ExeUtilExpr(ExeUtilType type, @@ -1030,12 +1031,7 @@ public: NABoolean noLog = FALSE, NABoolean ignoreTrigger = FALSE, NABoolean isPurgedata = FALSE, - CollHeap *oHeap = CmpCommon::statementHeap(), - NABoolean isHiveTable = FALSE, - NAString * hiveTableLocation = NULL, - NAString * hiveHostName = NULL, - Int32 hiveHdfsPort = 0, - Int64 hiveModTS = -1) + CollHeap *oHeap = CmpCommon::statementHeap()) : ExeUtilExpr(FAST_DELETE_, name, exprNode, NULL, stmtText, stmtTextCharSet, oHeap), doPurgedataCat_(doPurgedataCat), noLog_(noLog), ignoreTrigger_(ignoreTrigger), @@ -1044,18 +1040,8 @@ public: doParallelDeleteIfXn_(FALSE), offlineTable_(FALSE), doLabelPurgedata_(FALSE), - numLOBs_(0), - isHiveTable_(isHiveTable), - hiveModTS_(hiveModTS) + numLOBs_(0) { - if (isHiveTable ) - { - CMPASSERT(hiveTableLocation != NULL); - hiveTableLocation_ = *hiveTableLocation; - if (hiveHostName) - hiveHostName_ = *hiveHostName; - hiveHdfsPort_ = hiveHdfsPort; - } }; virtual NABoolean isExeUtilQueryType() { return TRUE; } @@ -1074,27 +1060,6 @@ public: virtual NABoolean aqrSupported() { return TRUE; } - - NABoolean isHiveTable() - { - return isHiveTable_; - } - - const NAString &getHiveTableLocation() const - { - return hiveTableLocation_; - } - - const NAString &getHiveHostName() const - { - return hiveHostName_; - } - - const Int32 getHiveHdfsPort() const - { - return hiveHdfsPort_; - } - private: NABoolean doPurgedataCat_; @@ -1120,14 +1085,63 @@ private: // if there are LOB columns. Lng32 numLOBs_; // number of LOB columns NAList<short> lobNumArray_; // array of shorts. Each short is the lob num +}; - NABoolean isHiveTable_; +class ExeUtilHiveTruncate : public ExeUtilExpr +{ +public: + ExeUtilHiveTruncate(const CorrName &name, + ConstStringList * pl, + CollHeap *oHeap = CmpCommon::statementHeap()) + : ExeUtilExpr(HIVE_TRUNCATE_, name, NULL, NULL, NULL, + CharInfo::UnknownCharSet, oHeap), + pl_(pl) + { + }; + + virtual NABoolean isExeUtilQueryType() { return TRUE; } + + virtual RelExpr * copyTopNode(RelExpr *derivedNode = NULL, + CollHeap* outHeap = 0); + + virtual RelExpr * bindNode(BindWA *bindWAPtr); + + virtual RelExpr * preCodeGen(Generator * generator, + const ValueIdSet & externalInputs, + ValueIdSet &pulledNewInputs); + + // method to do code generation + virtual short codeGen(Generator*); + + virtual NABoolean aqrSupported() { return FALSE; } + + const NAString &getHiveTableLocation() const + { + return hiveTableLocation_; + } + + const NAString &getHiveHostName() const + { + return hiveHostName_; + } + + const Int32 getHiveHdfsPort() const + { + return hiveHdfsPort_; + } + + ConstStringList* &partnList() { return pl_; } + +private: NAString hiveTableLocation_; NAString hiveHostName_; Int32 hiveHdfsPort_; // timestamp of hiveTableLocation. Int64 hiveModTS_; + + // list of partitions to be truncated + ConstStringList * pl_; }; class ExeUtilMaintainObject : public ExeUtilExpr http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/parser/sqlparser.y ---------------------------------------------------------------------- diff --git a/core/sql/parser/sqlparser.y b/core/sql/parser/sqlparser.y index dedbe67..6587ae3 100755 --- a/core/sql/parser/sqlparser.y +++ b/core/sql/parser/sqlparser.y @@ -2840,6 +2840,7 @@ static void enableMakeQuotedStringISO88591Mechanism() %type <relx> exe_util_fast_delete %type <longint> purgedata_options +%type <relx> exe_util_hive_truncate %type <relx> exe_util_get_metadata_info %type <relx> exe_util_get_version_info @@ -14550,6 +14551,10 @@ interactive_query_expression: { $$ = finalize($1); } + | exe_util_hive_truncate + { + $$ = finalize($1); + } | exe_util_get_uid { $$ = finalize($1); @@ -17380,11 +17385,6 @@ optional_mt_options : QUOTED_STRING exe_util_fast_delete: TOK_PURGEDATA table_name purgedata_options { - if (CmpCommon::getDefault(EXE_PARALLEL_PURGEDATA) == DF_OFF) - { - YYERROR; - } - short noLog = ($3 & 0x1) != 0; short ignoreTrigger = ($3 & 0x2) != 0; @@ -17406,8 +17406,6 @@ exe_util_fast_delete: TOK_PURGEDATA table_name purgedata_options FALSE, noLog, ignoreTrigger, - // (($3 == 1) || ($3 == 3)), - // (($3 == 2) || ($3 == 3)), TRUE, PARSERHEAP()); @@ -17427,6 +17425,24 @@ purgedata_options : /*empty*/ { $$ = 0; } | TOK_NOLOG TOK_WAITEDIO TOK_IGNORE_TRIGGER { $$ = 7; } | TOK_IGNORE_TRIGGER TOK_WAITEDIO TOK_NOLOG { $$ = 7; } +exe_util_hive_truncate: TOK_TRUNCATE table_name + { + $$ = new (PARSERHEAP()) + ExeUtilHiveTruncate(CorrName(*$2, PARSERHEAP()), + NULL, + PARSERHEAP()); + + delete $2; + } + | TOK_TRUNCATE table_name TOK_PARTITION '(' quoted_string_list ')' + { + $$ = new (PARSERHEAP()) + ExeUtilHiveTruncate(CorrName(*$2, PARSERHEAP()), + $5, + PARSERHEAP()); + + delete $2; + } exe_util_aqr: TOK_GET TOK_ALL TOK_AQR TOK_ENTRIES { @@ -17955,6 +17971,13 @@ hbb_unload_option: hbb_unload_empty_target new (PARSERHEAP ()) UnloadOption(UnloadOption::EMPTY_TARGET_,0,NULL); $$ = op; } + | TOK_TRUNCATE TOK_FROM TOK_TARGET + { + //purge target folder + UnloadOption *op = + new (PARSERHEAP ()) UnloadOption(UnloadOption::EMPTY_TARGET_,0,NULL); + $$ = op; + } hbb_unload_compress: TOK_COMPRESSION TOK_GZIP { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/regress/hive/EXPECTED005 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED005 b/core/sql/regress/hive/EXPECTED005 index 0c505e3..e134db1 100644 --- a/core/sql/regress/hive/EXPECTED005 +++ b/core/sql/regress/hive/EXPECTED005 @@ -23,7 +23,6 @@ >>-- getting a query cache hit and updating the changed >>-- Hive and HDFS metadata >>------------------------------------------------------------ ->> >>prepare s1 from +> select c_preferred_cust_flag, +> count(*) @@ -434,9 +433,9 @@ ID CHAPTER ENGLISH TRANSLATOR >>select * from tbl_type; TINT SM I BIG STR F D T DT VC ------- ------ ----------- -------------------- ------------------------- --------------- ------------------------- -------------------------- ---------- ---------- +------ ------ ----------- -------------------- ------------------------- --------------- ------------------------- -------------------------- ---------- ---------------------------------------- - 201 202 203 204 two hundred 2.0000000E+002 2.00000000000000000E+002 2022-02-02 22:22:22.222222 2022-02-02 varchar + 201 202 203 204 two hundred 2.0000000E+002 2.00000000000000000E+002 2022-02-02 22:22:22.222222 2022-02-02 varchar --- 1 row(s) selected. >>insert into tbl_type_temp select * from tbl_type; @@ -445,9 +444,9 @@ TINT SM I BIG STR F >>select * from tbl_type_temp; TINT SM I BIG STR F D T DT VC ------- ------ ----------- -------------------- ------------------------- --------------- ------------------------- -------------------------- ---------- ---------- +------ ------ ----------- -------------------- ------------------------- --------------- ------------------------- -------------------------- ---------- ---------------------------------------- - 201 202 203 204 two hundred 2.0000000E+002 2.00000000000000000E+002 2022-02-02 22:22:22.222222 2022-02-02 varchar + 201 202 203 204 two hundred 2.0000000E+002 2.00000000000000000E+002 2022-02-02 22:22:22.222222 2022-02-02 varchar --- 1 row(s) selected. >> @@ -710,11 +709,11 @@ A B --- 1 row(s) selected. >> ->>-- purgedata of hive data +>>-- truncate of hive data >>cqd query_cache '0'; --- SQL operation complete. ->>purgedata hive.hive.thive; +>>truncate hive.hive.thive; --- SQL operation complete. >>select * from hive.hive.thive; @@ -731,14 +730,96 @@ A B 10 20 --- 1 row(s) selected. ->>purgedata hive.hive.thive; +>>truncate hive.hive.thive; --- SQL operation complete. ->>purgedata hive.hive.thive; +>>truncate hive.hive.thive; --- SQL operation complete. >>select * from hive.hive.thive; --- 0 row(s) selected. >> +>>-- truncate of partitioned hive table +>>sh echo "drop table t005part;" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk; +>>sh echo "create table t005part(a int) partitioned by (b int, c int);" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk; +>> +>>sh echo "insert into t005part partition (b=10,c=11) values (5);" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk; +>>sh echo "select * from t005part;" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk | tee -a LOG005; +t005part.a t005part.b t005part.c +5 10 11 +>> +>>truncate hive.hive.t005part; + +--- SQL operation complete. +>>sh echo "select * from t005part;" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk | tee -a LOG005; +t005part.a t005part.b t005part.c +>> +>>sh echo "insert into t005part partition (b=10,c=11) values (5);" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk; +>>truncate hive.hive.t005part partition ('b=10'); + +--- SQL operation complete. +>>sh echo "select * from t005part;" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk | tee -a LOG005; +t005part.a t005part.b t005part.c +>> +>> +>>sh echo "insert into t005part partition (b=10,c=11) values (5);" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk; +>>truncate hive.hive.t005part partition ('b=10','c=11'); + +--- SQL operation complete. +>>sh echo "select * from t005part;" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk | tee -a LOG005; +t005part.a t005part.b t005part.c +>> +>>sh echo "insert into t005part partition (b=10,c=11) values (5);" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk; +>>sh echo "insert into t005part partition (b=10,c=12) values (6);" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk; +>>sh echo "select * from t005part;" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk | tee -a LOG005; +t005part.a t005part.b t005part.c +5 10 11 +6 10 12 +>> +>>truncate hive.hive.t005part partition ('b=10','c=11'); + +--- SQL operation complete. +>>sh echo "select * from t005part;" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk | tee -a LOG005; +t005part.a t005part.b t005part.c +6 10 12 +>> +>>truncate hive.hive.t005part partition ('b=10'); + +--- SQL operation complete. +>>sh echo "select * from t005part;" > TEST005_junk; +>>sh regrhive.ksh -f TEST005_junk | tee -a LOG005; +t005part.a t005part.b t005part.c +>> +>>-- should return error +>>truncate hive.hive.t005part partition ('b=12'); + +*** ERROR[8442] Unable to access HDFS interface. Call to ExpLOBInterfaceEmptyDirectory returned error LOB_DIR_NAME_ERROR(535). Error detail 0. + +*** ERROR[8034] Truncation of hive table failed. Reason: specified partition does not exist + +--- SQL operation failed with errors. +>> +>> +>>-- should return error +>>purgedata hive.hive.thive; + +*** ERROR[3242] This statement is not supported. Reason: Purgedata is not allowed for hive tables. Use Truncate command. + +*** ERROR[8822] The statement was not prepared. + +>> >>log; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/regress/hive/TEST005 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST005 b/core/sql/regress/hive/TEST005 index 2fe0b86..a2402ce 100644 --- a/core/sql/regress/hive/TEST005 +++ b/core/sql/regress/hive/TEST005 @@ -69,7 +69,6 @@ cqd HIST_ROWCOUNT_REQUIRING_STATS '50000'; -- getting a query cache hit and updating the changed -- Hive and HDFS metadata ------------------------------------------------------------ - prepare s1 from select c_preferred_cust_flag, count(*) @@ -311,14 +310,64 @@ select a from hive.hive.thive; select * from hive.hive.thive; --- purgedata of hive data +-- truncate of hive data cqd query_cache '0'; -purgedata hive.hive.thive; +truncate hive.hive.thive; select * from hive.hive.thive; insert into hive.hive.thive values (10, 20); select * from hive.hive.thive; -purgedata hive.hive.thive; -purgedata hive.hive.thive; +truncate hive.hive.thive; +truncate hive.hive.thive; select * from hive.hive.thive; +-- truncate of partitioned hive table +sh echo "drop table t005part;" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk; +sh echo "create table t005part(a int) partitioned by (b int, c int);" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk; + +sh echo "insert into t005part partition (b=10,c=11) values (5);" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk; +sh echo "select * from t005part;" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk | tee -a LOG005; + +truncate hive.hive.t005part; +sh echo "select * from t005part;" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk | tee -a LOG005; + +sh echo "insert into t005part partition (b=10,c=11) values (5);" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk; +truncate hive.hive.t005part partition ('b=10'); +sh echo "select * from t005part;" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk | tee -a LOG005; + + +sh echo "insert into t005part partition (b=10,c=11) values (5);" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk; +truncate hive.hive.t005part partition ('b=10','c=11'); +sh echo "select * from t005part;" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk | tee -a LOG005; + +sh echo "insert into t005part partition (b=10,c=11) values (5);" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk; +sh echo "insert into t005part partition (b=10,c=12) values (6);" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk; +sh echo "select * from t005part;" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk | tee -a LOG005; + +truncate hive.hive.t005part partition ('b=10','c=11'); +sh echo "select * from t005part;" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk | tee -a LOG005; + +truncate hive.hive.t005part partition ('b=10'); +sh echo "select * from t005part;" > TEST005_junk; +sh regrhive.ksh -f TEST005_junk | tee -a LOG005; + +-- should return error +truncate hive.hive.t005part partition ('b=12'); + + +-- should return error +purgedata hive.hive.thive; + log; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/sqlci/sqlci_lex.ll ---------------------------------------------------------------------- diff --git a/core/sql/sqlci/sqlci_lex.ll b/core/sql/sqlci/sqlci_lex.ll index 38753a3..4f8b4a7 100755 --- a/core/sql/sqlci/sqlci_lex.ll +++ b/core/sql/sqlci/sqlci_lex.ll @@ -446,6 +446,7 @@ B [ \t\n]+ [Ii][Nn][Tt][Ee][Rr][Nn][Aa][Ll] return_IDENT_or_TOKEN(INTERNAL, 0); /* MV OZ_REFRESH */ [Mm][Vv][Ll][Oo][Gg] return_IDENT_or_TOKEN(MVLOG, 0); [Uu][Nn][Ll][Oo][Aa][Dd] return_IDENT_or_TOKEN(UNLOAD, 0); +[Tt][Rr][Uu][Nn][Cc][Aa][Tt][Ee] return_IDENT_or_TOKEN(TRUNCATE, 0); [\*] {SqlciParse_IdentifierExpected = 0; return(ALLtoken);}; [(] {SqlciParse_IdentifierExpected = 0; return(LPAREN);}; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/sqlci/sqlci_yacc.y ---------------------------------------------------------------------- diff --git a/core/sql/sqlci/sqlci_yacc.y b/core/sql/sqlci/sqlci_yacc.y index 972e4d1..6b92a95 100644 --- a/core/sql/sqlci/sqlci_yacc.y +++ b/core/sql/sqlci/sqlci_yacc.y @@ -509,6 +509,7 @@ static char * FCString (const char *idString, int isFC) %token SET_TRANSACTIONtoken %token TERMINAL_CHARSET %token TRANSFORM +%token TRUNCATE %token UNLOCK %token UPD_STATS %token UPD_HIST_STATS @@ -2229,6 +2230,7 @@ dml_type : | MVLOG {$$ = DML_CONTROL_TYPE;} | DUP {$$ = DML_DDL_TYPE;} | PURGEDATA {$$ = DML_DDL_TYPE;} + | TRUNCATE {$$ = DML_DDL_TYPE;} | POPULATE {$$ = DML_DDL_TYPE;} | VALIDATEtoken {$$ = DML_DDL_TYPE;} | RECOVER {$$ = DML_DDL_TYPE;} http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/sqlcomp/DefaultConstants.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h index aa90c7c..e92cfad 100644 --- a/core/sql/sqlcomp/DefaultConstants.h +++ b/core/sql/sqlcomp/DefaultConstants.h @@ -2677,10 +2677,6 @@ enum DefaultConstants UNIQUE_HASH_JOIN_MAX_INNER_SIZE_PER_INSTANCE, UNIQUE_HASH_JOIN_MAX_INNER_TABLES, - // this will turn ON purgedata using ESP parallelism. - EXE_PARALLEL_PURGEDATA, - - // catalog where maintain database is maintained MAINTAIN_CATALOG, @@ -2768,8 +2764,6 @@ enum DefaultConstants // duplicated. The value used must be an UNQUALIFIED table name. USTAT_SAMPLE_TABLE_NAME_CREATE, - EXE_PARALLEL_PURGEDATA_WARNINGS, - // By default (ON), distribute the metadata tables across all // segments. If set to LOCAL_NODE, distribute metadata tables // across volumes in the local segment when the first schema is http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e7a1dc0d/core/sql/sqlcomp/nadefaults.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp index 9ecfd29..27607c6 100644 --- a/core/sql/sqlcomp/nadefaults.cpp +++ b/core/sql/sqlcomp/nadefaults.cpp @@ -1354,10 +1354,6 @@ SDDkwd__(EXE_DIAGNOSTIC_EVENTS, "OFF"), DDkwd__(EXE_PARALLEL_DDL, "ON"), - DDkwd__(EXE_PARALLEL_PURGEDATA, "MINIMUM"), - - DDkwd__(EXE_PARALLEL_PURGEDATA_WARNINGS, "OFF"), - DDui___(EXE_PA_DP2_STATIC_AFFINITY, "1"), DDkwd__(EXE_SINGLE_BMO_QUOTA, "ON"), @@ -6673,12 +6669,6 @@ DefaultToken NADefaults::token(Int32 attrEnum, isValid = TRUE; break; - case EXE_PARALLEL_PURGEDATA: - if (tok == DF_ALL || tok == DF_MINIMUM || - tok == DF_OFF || tok == DF_ON || tok == DF_MEDIUM) - isValid = TRUE; - break; - case HIDE_INDEXES: if (tok == DF_NONE || tok == DF_ALL || tok == DF_VERTICAL || tok == DF_INDEXES || tok == DF_KEYINDEXES)
