hive data modification detection: commit #4
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/51a9c73e Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/51a9c73e Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/51a9c73e Branch: refs/heads/master Commit: 51a9c73ed37c3711178483c4a200f17a92d1c08d Parents: b1a8f02 Author: Anoop Sharma <[email protected]> Authored: Tue May 31 18:03:40 2016 +0000 Committer: Anoop Sharma <[email protected]> Committed: Tue May 31 18:03:40 2016 +0000 ---------------------------------------------------------------------- core/sql/cli/SessionDefaults.cpp | 2 + core/sql/comexe/ComTdbExeUtil.cpp | 6 +- core/sql/comexe/ComTdbExeUtil.h | 12 ++- core/sql/comexe/ComTdbHdfsScan.h | 4 +- core/sql/executor/ExExeUtil.h | 9 +- core/sql/executor/ExExeUtilMisc.cpp | 95 ++++++++++++++----- core/sql/executor/ExFastTransport.cpp | 2 +- core/sql/executor/ExHdfsScan.cpp | 2 +- core/sql/exp/ExpErrorEnums.h | 1 + core/sql/exp/ExpLOBaccess.cpp | 133 ++++----------------------- core/sql/exp/ExpLOBaccess.h | 5 +- core/sql/exp/ExpLOBinterface.h | 5 +- core/sql/generator/GenFastTransport.cpp | 2 +- core/sql/generator/GenRelExeUtil.cpp | 5 +- core/sql/optimizer/BindRelExpr.cpp | 3 +- core/sql/optimizer/HDFSHook.cpp | 50 ++++------ core/sql/optimizer/NATable.cpp | 3 + core/sql/optimizer/RelExeUtil.h | 9 +- core/sql/optimizer/RelFastTransport.cpp | 1 + core/sql/regress/hive/EXPECTED003 | 37 ++++++-- core/sql/regress/hive/EXPECTED005 | 36 +++++--- core/sql/regress/hive/EXPECTED015 | 59 ++++++------ core/sql/regress/hive/TEST003 | 9 +- core/sql/regress/hive/TEST005 | 11 +-- core/sql/regress/hive/TEST015 | 3 +- 25 files changed, 240 insertions(+), 264 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/cli/SessionDefaults.cpp ---------------------------------------------------------------------- diff --git a/core/sql/cli/SessionDefaults.cpp b/core/sql/cli/SessionDefaults.cpp index 024754b..5e79138 100644 --- a/core/sql/cli/SessionDefaults.cpp +++ b/core/sql/cli/SessionDefaults.cpp @@ -782,6 +782,8 @@ static const AQRInfo::AQRErrorMap aqrErrorMap[] = // parallel purgedata failed AQREntry( 8022, 0, 3, 60, 0, 0, "", 0, 1), + // hive data modification timestamp mismatch. + // query will be AQR'd and hive metadata will be reloaded. AQREntry( 8436, 0, 1, 0, 0, 2, "04:05", 0, 0), // FS memory errors http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/comexe/ComTdbExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbExeUtil.cpp b/core/sql/comexe/ComTdbExeUtil.cpp index 3442714..e3dd630 100644 --- a/core/sql/comexe/ComTdbExeUtil.cpp +++ b/core/sql/comexe/ComTdbExeUtil.cpp @@ -1189,7 +1189,8 @@ ComTdbExeUtilFastDelete::ComTdbExeUtilFastDelete( NABoolean isHiveTruncate, char * hiveTableLocation, char * hiveHostName, - Lng32 hivePortNum) + Lng32 hivePortNum, + Int64 hiveModTS) : ComTdbExeUtil(ComTdbExeUtil::FAST_DELETE_, NULL, 0, (Int16)SQLCHARSETCODE_UNKNOWN, tableName, tableNameLen, @@ -1211,7 +1212,8 @@ ComTdbExeUtilFastDelete::ComTdbExeUtilFastDelete( lobNumArray_(lobNumArray), hiveTableLocation_(hiveTableLocation), hiveHdfsHost_(hiveHostName), - hiveHdfsPort_(hivePortNum) + hiveHdfsPort_(hivePortNum), + hiveModTS_(hiveModTS) { setIsHiveTruncate(isHiveTruncate); setNodeType(ComTdb::ex_FAST_DELETE); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/comexe/ComTdbExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbExeUtil.h b/core/sql/comexe/ComTdbExeUtil.h index ea3ef69..eb70eac 100644 --- a/core/sql/comexe/ComTdbExeUtil.h +++ b/core/sql/comexe/ComTdbExeUtil.h @@ -1518,7 +1518,8 @@ public: NABoolean ishiveTruncate = FALSE, char * hiveTableLocation = NULL, char * hiveHostName = NULL, - Lng32 hivePortNum = 0 + Lng32 hivePortNum = 0, + Int64 hiveModTS = -1 ); Long pack(void *); @@ -1569,6 +1570,11 @@ public: return hiveHdfsPort_; } + Lng32 getHiveModTS() const + { + return hiveModTS_; + } + // --------------------------------------------------------------------- // Used by the internal SHOWPLAN command to get attributes of a TDB. // --------------------------------------------------------------------- @@ -1645,7 +1651,9 @@ private: NABasicPtr hiveTableLocation_; // 56-63 NABasicPtr hiveHdfsHost_; // 64-71 Int32 hiveHdfsPort_; // 72-75 - char fillersComTdbExeUtilFastDelete_[52]; // 76-127 + char fillers1_[4]; // 76-79 + Int64 hiveModTS_; // 80-87 + char fillersComTdbExeUtilFastDelete_[40]; // 88-127 }; class ComTdbExeUtilGetStatistics : public ComTdbExeUtil http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/comexe/ComTdbHdfsScan.h ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbHdfsScan.h b/core/sql/comexe/ComTdbHdfsScan.h index ac83311..392f7cf 100755 --- a/core/sql/comexe/ComTdbHdfsScan.h +++ b/core/sql/comexe/ComTdbHdfsScan.h @@ -135,7 +135,7 @@ class ComTdbHdfsScan : public ComTdb char fillersComTdbHdfsScan1_[2]; // 190 - 191 NABasicPtr nullFormat_; // 192 - 199 - // next 3 params used to check if data under hdfsFileDir + // next 4 params are used to check if data under hdfsFileDir // was modified after query was compiled. NABasicPtr hdfsRootDir_; // 200 - 207 Int64 modTSforDir_; // 208 - 215 @@ -196,7 +196,7 @@ public: char * loggingLocation = NULL, char * errCountId = NULL, - // next 3 params used to check if data under hdfsFileDir + // next 4 params are used to check if data under hdfsFileDir // was modified after query was compiled. char * hdfsRootDir = NULL, Int64 modTSforDir = -1, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/executor/ExExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h index 6ed29c1..062362b 100755 --- a/core/sql/executor/ExExeUtil.h +++ b/core/sql/executor/ExExeUtil.h @@ -3299,6 +3299,7 @@ class ExExeUtilHiveTruncateTcb : public ExExeUtilTcb { INITIAL_, ERROR_, + DATA_MOD_CHECK_, EMPTY_DIRECTORY_, DONE_ }; @@ -3306,18 +3307,10 @@ class ExExeUtilHiveTruncateTcb : public ExExeUtilTcb ExExeUtilFastDeleteTdb & fdTdb() const {return (ExExeUtilFastDeleteTdb &) tdb;}; - -// short doHiveTruncate(char * objectName, -// NABoolean isIndex, -// NABoolean fastDelUsingResetEOF); - short injectError(const char * val); Step step_; - char hdfsHost_[500]; - int hdfsPort_; - char hiveTableLocation_[513]; int numExistingFiles_; void * lobGlob_; }; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/executor/ExExeUtilMisc.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtilMisc.cpp b/core/sql/executor/ExExeUtilMisc.cpp index 1aea6c1..267d22e 100644 --- a/core/sql/executor/ExExeUtilMisc.cpp +++ b/core/sql/executor/ExExeUtilMisc.cpp @@ -2240,13 +2240,7 @@ ExExeUtilHiveTruncateTcb::ExExeUtilHiveTruncateTcb( qparent_.down->allocatePstate(this); numExistingFiles_ = 0; - memset (hdfsHost_, '\0', sizeof(hdfsHost_)); - memset (hiveTableLocation_, '\0', sizeof(hiveTableLocation_)); - strncpy(hdfsHost_, fdTdb().getHiveHdfsHost(), sizeof(hdfsHost_)); - hdfsPort_ = fdTdb().getHiveHdfsPort(); - char * outputPath = fdTdb().getHiveTableLocation(); - strncpy(hiveTableLocation_, outputPath, 512); step_ = INITIAL_; } @@ -2290,26 +2284,84 @@ short ExExeUtilHiveTruncateTcb::work() case INITIAL_: { - //nothing for now -- - // more stuff later + if (fdTdb().getHiveModTS() > 0) + step_ = DATA_MOD_CHECK_; + else + step_ = EMPTY_DIRECTORY_; + } + break; + + case DATA_MOD_CHECK_: + { + cliRC = ExpLOBinterfaceDataModCheck + (lobGlob_, + fdTdb().getHiveTableLocation(), + fdTdb().getHiveHdfsHost(), + fdTdb().getHiveHdfsPort(), + fdTdb().getHiveModTS(), + 0); + + if (cliRC < 0) + { + Lng32 cliError = 0; + + Lng32 intParam1 = -cliRC; + ComDiagsArea * diagsArea = NULL; + ExRaiseSqlError(getHeap(), &diagsArea, + (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE), + NULL, &intParam1, + &cliError, + NULL, + "HDFS", + (char*)"ExpLOBInterfaceEmptyDirectory", + getLobErrStr(intParam1)); + pentry_down->setDiagsArea(diagsArea); + step_ = ERROR_; + break; + } + + if (cliRC == 1) // data mod check failed + { + ComDiagsArea * diagsArea = NULL; + ExRaiseSqlError(getHeap(), &diagsArea, + (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR)); + pentry_down->setDiagsArea(diagsArea); + + step_ = ERROR_; + break; + } + step_ = EMPTY_DIRECTORY_; } - break; + break; case EMPTY_DIRECTORY_: { - Lng32 retCode= ExpLOBinterfaceEmptyDirectory( - lobGlob_, - (char*)"", //name is empty - hiveTableLocation_, - Lob_HDFS_File, - hdfsHost_, - hdfsPort_, - 0 , - 1 , - 0); - if (retCode != 0) + cliRC = ExpLOBinterfaceEmptyDirectory( + lobGlob_, + (char*)"", //name is empty + fdTdb().getHiveTableLocation(), + Lob_HDFS_File, + fdTdb().getHiveHdfsHost(), + fdTdb().getHiveHdfsPort(), + 0 , + 1 , + 0); + if (cliRC != 0) { + Lng32 cliError = 0; + + Lng32 intParam1 = -cliRC; + ComDiagsArea * diagsArea = NULL; + ExRaiseSqlError(getHeap(), &diagsArea, + (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE), + NULL, &intParam1, + &cliError, + NULL, + "HDFS", + (char*)"ExpLOBInterfaceEmptyDirectory", + getLobErrStr(intParam1)); + pentry_down->setDiagsArea(diagsArea); step_ = ERROR_; } else @@ -2318,6 +2370,7 @@ short ExExeUtilHiveTruncateTcb::work() } } break; + case ERROR_: { if (qparent_.up->isFull()) @@ -2350,7 +2403,7 @@ short ExExeUtilHiveTruncateTcb::work() step_ = DONE_; } - break; + break; case DONE_: { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/executor/ExFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExFastTransport.cpp b/core/sql/executor/ExFastTransport.cpp index 45c3959..bb8d4dc 100644 --- a/core/sql/executor/ExFastTransport.cpp +++ b/core/sql/executor/ExFastTransport.cpp @@ -738,7 +738,7 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work() { ComDiagsArea * diagsArea = NULL; ExRaiseSqlError(getHeap(), &diagsArea, - (ExeErrorCode)(8436)); + (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR)); pentry_down->setDiagsArea(diagsArea); pstate.step_ = EXTRACT_ERROR; break; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/executor/ExHdfsScan.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp index b204fb2..82e01b3 100644 --- a/core/sql/executor/ExHdfsScan.cpp +++ b/core/sql/executor/ExHdfsScan.cpp @@ -468,7 +468,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work() { ComDiagsArea * diagsArea = NULL; ExRaiseSqlError(getHeap(), &diagsArea, - (ExeErrorCode)(8436)); + (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR)); pentry_down->setDiagsArea(diagsArea); step_ = HANDLE_ERROR_AND_DONE; break; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/exp/ExpErrorEnums.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpErrorEnums.h b/core/sql/exp/ExpErrorEnums.h index de4d00f..987fe7e 100644 --- a/core/sql/exp/ExpErrorEnums.h +++ b/core/sql/exp/ExpErrorEnums.h @@ -153,6 +153,7 @@ enum ExeErrorCode EXE_IS_BITWISE_AND_ERROR = 8431, EXE_UNSIGNED_OVERFLOW = 8432, EXE_INVALID_CHARACTER = 8433, + EXE_HIVE_DATA_MOD_CHECK_ERROR = 8436, EXE_HISTORY_BUFFER_TOO_SMALL = 8440, EXE_OLAP_OVERFLOW_NOT_SUPPORTED = 8441, EXE_ERROR_FROM_LOB_INTERFACE = 8442, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/exp/ExpLOBaccess.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp index 7db4a40..ad127ea 100644 --- a/core/sql/exp/ExpLOBaccess.cpp +++ b/core/sql/exp/ExpLOBaccess.cpp @@ -110,104 +110,6 @@ ExLob::~ExLob() } -#ifdef __ignore -Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode, - char *dir, - LobsStorage storage, - char *hdfsServer, Int64 hdfsPort, - char *lobLocation, - int bufferSize , short replication , - int blockSize, Int64 lobMaxSize, - ExLobGlobals *lobGlobals) -{ - int openFlags; - mode_t filePerms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; - struct timespec startTime; - struct timespec endTime; - Int64 secs, nsecs, totalnsecs; - - if (dir) - { - if (dir_.empty()) - { - dir_ = string(dir); - } - - if (lobFile) - snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s/%s", dir_.c_str(), lobFile); - - } - else if (lobFile) - { - snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s", lobFile); - - } - - hdfsServer_ = hdfsServer; - hdfsPort_ = hdfsPort; - - if (fs_ == NULL) - { - fs_ = hdfsConnect(hdfsServer_, hdfsPort_); - if (fs_ == NULL) - return LOB_HDFS_CONNECT_ERROR; - } - - if (lobGlobals) - lobGlobals->setHdfsFs(fs_); - - if (storage_ != Lob_Invalid_Storage) - { - return LOB_INIT_ERROR; - } - else - { - storage_ = storage; - } - - stats_.init(); - - if (lobLocation) - lobLocation_ = lobLocation; - clock_gettime(CLOCK_MONOTONIC, &startTime); - - clock_gettime(CLOCK_MONOTONIC, &endTime); - - secs = endTime.tv_sec - startTime.tv_sec; - nsecs = endTime.tv_nsec - startTime.tv_nsec; - if (nsecs < 0) - { - secs--; - nsecs += NUM_NSECS_IN_SEC; - } - totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs; - stats_.hdfsConnectionTime += totalnsecs; - - if (mode == EX_LOB_CREATE) - { - // check if file is already created - hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_); - if (fInfo != NULL) - { - hdfsFreeFileInfo(fInfo, 1); - return LOB_DATA_FILE_CREATE_ERROR; - } - openFlags = O_WRONLY | O_CREAT; - fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags, bufferSize, replication, blockSize); - if (!fdData_) - { - return LOB_DATA_FILE_CREATE_ERROR; - } - hdfsCloseFile(fs_, fdData_); - fdData_ = NULL; - - } - lobGlobalHeap_ = lobGlobals->getHeap(); - return LOB_OPER_OK; - -} -#endif - Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode, char *dir, LobsStorage storage, @@ -573,34 +475,34 @@ Ex_Lob_Error ExLob::emptyDirectory() int numExistingFiles=0; hdfsFileInfo *fileInfos = hdfsGetPathInfo(fs_, lobDataFile_); if (fileInfos == NULL) - { - return LOB_DATA_FILE_NOT_FOUND_ERROR; //here a directory - } + { + return LOB_DIR_NAME_ERROR; + } fileInfos = hdfsListDirectory(fs_, lobDataFile_, &numExistingFiles); - if (fileInfos == NULL) - { - return LOB_OPER_OK; - } - - for (int i = 0; i < numExistingFiles; i++) + if (fileInfos == NULL) // empty directory { -#ifdef USE_HADOOP_1 - int retCode = hdfsDelete(fs_, fileInfos[i].mName); -#else - int retCode = hdfsDelete(fs_, fileInfos[i].mName, 0); -#endif + return LOB_OPER_OK; + } + + NABoolean error = FALSE; + for (int i = 0; ((NOT error) && (i < numExistingFiles)); i++) + { + // if dir, recursively delete it and everything under it + int retCode = hdfsDelete(fs_, fileInfos[i].mName, 1); if (retCode !=0) { - //ex_assert(retCode == 0, "delete returned error"); - return LOB_DATA_FILE_DELETE_ERROR; + error = TRUE; } } + if (fileInfos) { hdfsFreeFileInfo(fileInfos, numExistingFiles); } - + + if (error) + return LOB_DATA_FILE_DELETE_ERROR; return LOB_OPER_OK; } @@ -2274,6 +2176,7 @@ Ex_Lob_Error ExLobsOper ( if (globPtr == NULL) { if ((operation == Lob_Init) || + (operation == Lob_Empty_Directory) || (operation == Lob_Data_Mod_Check)) { globPtr = (void *) new ExLobGlobals(); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/exp/ExpLOBaccess.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h index 416529d..2cce09e 100644 --- a/core/sql/exp/ExpLOBaccess.h +++ b/core/sql/exp/ExpLOBaccess.h @@ -481,10 +481,7 @@ class ExLob // dirPath: path to needed directory (includes directory name) // modTS is the latest timestamp on any file/dir under dirPath. - // numFilesInDir is the total number of files under dirPath. - // This method validates that current modTS is not greater then input modTS - // and current number of files in dirPath are the same as input numFilesInDir. - // If either condition is not true, then check fails. + // This method validates that current modTS is not greater then input modTS. // Return: LOB_OPER_OK, if passes. LOB_DATA_MOD_CHECK_ERROR, if fails. Ex_Lob_Error dataModCheck( char * dirPath, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/exp/ExpLOBinterface.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBinterface.h b/core/sql/exp/ExpLOBinterface.h index 689e422..859417e 100644 --- a/core/sql/exp/ExpLOBinterface.h +++ b/core/sql/exp/ExpLOBinterface.h @@ -293,10 +293,7 @@ Lng32 ExpLOBinterfacePurgeBackupLobDataFile(void *& lobGlob, char *hdfsServer, // dirPath: path to needed directory (includes directory name) // modTS is the latest timestamp on any file/dir under dirPath. -// numFilesInDir is the total number of files under dirPath. -// This method validates that current modTS is not greater then input modTS -// and current number of files in dirPath are the same as input numFilesInDir. -// If either condition is not true, then check fails. +// This method validates that current modTS is not greater then input modTS. // Return: 1, if check fails. 0, if passes. -1, if error. Lng32 ExpLOBinterfaceDataModCheck(void * lobGlob, char * dirPath, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/generator/GenFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenFastTransport.cpp b/core/sql/generator/GenFastTransport.cpp index 7e1ee3e..6a4b4e7 100644 --- a/core/sql/generator/GenFastTransport.cpp +++ b/core/sql/generator/GenFastTransport.cpp @@ -624,7 +624,7 @@ PhysicalFastExtract::codeGen(Generator *generator) { newTdb->setIsHiveInsert(1); newTdb->setIncludeHeader(0); - setOverwriteHiveTable( getOverwriteHiveTable()); + newTdb->setOverwriteHiveTable( getOverwriteHiveTable()); } else { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/generator/GenRelExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelExeUtil.cpp b/core/sql/generator/GenRelExeUtil.cpp index 55eba37..7629657 100644 --- a/core/sql/generator/GenRelExeUtil.cpp +++ b/core/sql/generator/GenRelExeUtil.cpp @@ -3237,12 +3237,11 @@ short ExeUtilFastDelete::codeGen(Generator * generator) (ex_cri_desc *)(generator->getCriDesc(Generator::DOWN)), (queue_index)getDefault(GEN_DDL_SIZE_DOWN), (queue_index)getDefault(GEN_DDL_SIZE_UP), -#pragma nowarn(1506) // warning elimination getDefault(GEN_DDL_NUM_BUFFERS), getDefault(GEN_DDL_BUFFER_SIZE), isHiveTable(), - hiveTableLocation, hiveHdfsHost, hiveHdfsPort); -#pragma warn(1506) // warning elimination + hiveTableLocation, hiveHdfsHost, hiveHdfsPort, + hiveModTS_); if (doPurgedataCat_) exe_util_tdb->setDoPurgedataCat(TRUE); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/BindRelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp index b81fdcf..9c3f0c2 100644 --- a/core/sql/optimizer/BindRelExpr.cpp +++ b/core/sql/optimizer/BindRelExpr.cpp @@ -9195,7 +9195,8 @@ RelExpr *Insert::bindNode(BindWA *bindWA) TRUE, new (bindWA->wHeap()) NAString(tableDir), new (bindWA->wHeap()) NAString(hostName), - hdfsPort); + hdfsPort, + hTabStats->getModificationTS()); //new root to prevent error 4056 when binding newRelExpr = new (bindWA->wHeap()) RelRoot(newRelExpr); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/HDFSHook.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/HDFSHook.cpp b/core/sql/optimizer/HDFSHook.cpp index fda6611..0d6fd34 100644 --- a/core/sql/optimizer/HDFSHook.cpp +++ b/core/sql/optimizer/HDFSHook.cpp @@ -1070,59 +1070,41 @@ void HHDFSTableStats::print(FILE *ofd) fprintf(ofd,"====================================================================\n"); } -//extern __thread hdfsFS *globalFS; -hdfsFS *globalFS; - NABoolean HHDFSTableStats::connectHDFS(const NAString &host, Int32 port) { NABoolean result = TRUE; // establish connection to HDFS if needed - if (globalFS == NULL || - *globalFS == NULL || + if (fs_ == NULL || currHdfsHost_ != host || currHdfsPort_ != port) { - if (globalFS && *globalFS) - disconnectHDFS(); - - if (globalFS == NULL) + if (fs_) { - globalFS = new hdfsFS; - *globalFS = NULL; + hdfsDisconnect(fs_); + fs_ = NULL; } - - if (*globalFS == NULL) + fs_ = hdfsConnect(host, port); + + if (fs_ == NULL) { - *globalFS = hdfsConnect(host, port); + NAString errMsg("hdfsConnect to "); + errMsg += host; + errMsg += ":"; + errMsg += port; + errMsg += " failed"; + diags_.recordError(errMsg, "HHDFSTableStats::connectHDFS"); + result = FALSE; } - currHdfsHost_ = host; currHdfsPort_ = port; } - - fs_ = *globalFS; - if (fs_ == NULL) - { - NAString errMsg("hdfsConnect to "); - errMsg += host; - errMsg += ":"; - errMsg += port; - errMsg += " failed"; - diags_.recordError(errMsg, "HHDFSTableStats::connectHDFS"); - result = FALSE; - } - return result; } void HHDFSTableStats::disconnectHDFS() { - if (globalFS && *globalFS) - { - hdfsDisconnect(*globalFS); - *globalFS = NULL; - } - + if (fs_) + hdfsDisconnect(fs_); fs_ = NULL; } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/NATable.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp index 81eb5b0..66d18b6 100644 --- a/core/sql/optimizer/NATable.cpp +++ b/core/sql/optimizer/NATable.cpp @@ -7308,6 +7308,9 @@ NATable * NATableDB::get(const ExtendedQualName* key, BindWA* bindWA, NABoolean } } + // the reload cqd will be set during aqr after compiletime and runtime + // timestamp mismatch is detected. + // If set, reload hive metadata. if ((cachedNATable->isHiveTable()) && (CmpCommon::getDefault(HIVE_DATA_MOD_CHECK) == DF_ON) && (CmpCommon::getDefault(TRAF_RELOAD_NATABLE_CACHE) == DF_ON)) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/RelExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExeUtil.h b/core/sql/optimizer/RelExeUtil.h index 5d0150b..5c782e9 100644 --- a/core/sql/optimizer/RelExeUtil.h +++ b/core/sql/optimizer/RelExeUtil.h @@ -1034,7 +1034,8 @@ public: NABoolean isHiveTable = FALSE, NAString * hiveTableLocation = NULL, NAString * hiveHostName = NULL, - Int32 hiveHdfsPort = 0) + Int32 hiveHdfsPort = 0, + Int64 hiveModTS = -1) : ExeUtilExpr(FAST_DELETE_, name, exprNode, NULL, stmtText, stmtTextCharSet, oHeap), doPurgedataCat_(doPurgedataCat), noLog_(noLog), ignoreTrigger_(ignoreTrigger), @@ -1044,7 +1045,8 @@ public: offlineTable_(FALSE), doLabelPurgedata_(FALSE), numLOBs_(0), - isHiveTable_(isHiveTable) + isHiveTable_(isHiveTable), + hiveModTS_(hiveModTS) { if (isHiveTable ) { @@ -1123,6 +1125,9 @@ private: NAString hiveTableLocation_; NAString hiveHostName_; Int32 hiveHdfsPort_; + + // timestamp of hiveTableLocation. + Int64 hiveModTS_; }; class ExeUtilMaintainObject : public ExeUtilExpr http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/RelFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelFastTransport.cpp b/core/sql/optimizer/RelFastTransport.cpp index 45d4168..4734d2a 100644 --- a/core/sql/optimizer/RelFastTransport.cpp +++ b/core/sql/optimizer/RelFastTransport.cpp @@ -98,6 +98,7 @@ RelExpr * FastExtract::copyTopNode(RelExpr *derivedNode, result->recordSeparator_ = recordSeparator_ ; result->selectList_ = selectList_; result->isSequenceFile_ = isSequenceFile_; + result->overwriteHiveTable_ = overwriteHiveTable_; return RelExpr::copyTopNode(result, outHeap); } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/EXPECTED003 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED003 b/core/sql/regress/hive/EXPECTED003 index 79cdb5a..0d0c3d0 100644 --- a/core/sql/regress/hive/EXPECTED003 +++ b/core/sql/regress/hive/EXPECTED003 @@ -2,21 +2,18 @@ >>set schema hive.hive; --- SQL operation complete. ->>cqd attempt_esp_parallelism 'off'; - ---- SQL operation complete. ->>cqd hive_max_esps '1'; - ---- SQL operation complete. ->>cqd PARALLEL_NUM_ESPS '1'; - ---- SQL operation complete. +>>--cqd attempt_esp_parallelism 'off'; +>>--cqd hive_max_esps '1'; +>>--cqd PARALLEL_NUM_ESPS '1'; >>cqd HIVE_MAX_STRING_LENGTH '25' ; --- SQL operation complete. >>cqd mode_seahive 'ON'; --- SQL operation complete. +>>cqd auto_query_retry_warnings 'ON'; + +--- SQL operation complete. >> >>prepare s from insert into hive.ins_customer select * from hive.customer; @@ -135,6 +132,10 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S *** ERROR[8822] The statement was not prepared. +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[4023] The degree of each row value constructor (20) must equal the degree of the target table column list (19). + >>-- number of columns doesn't match >> >>prepare s from @@ -144,6 +145,10 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S *** ERROR[8822] The statement was not prepared. +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[4039] Column T_TIME_SK is of type INTEGER, incompatible with the value's type, CHAR(1). + >>-- wrong data types >> >> @@ -172,6 +177,10 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S >> >>insert OVERWRITE TABLE hive.ins_customer_address select * from >>hive.customer_address; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. + --- 50000 row(s) inserted. >> >>select count(*) from hive.customer_address; @@ -193,6 +202,10 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S >>--execute again >>insert OVERWRITE TABLE hive.ins_customer_address select * from >>hive.customer_address; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. + --- 50000 row(s) inserted. >> >>select count(*) from hive.customer_address; @@ -595,7 +608,7 @@ test lp bug # 1355477 > SS_SOLD_DATE_SK SS_STORE_SK SS_QUANTITY > --------------- ----------- ----------- >>--execute again --overwrite should get rid og existing data from previous run ->>control query shape union(cut,esp_exchange(cut)); +>>control query shape union(cut, esp_exchange(cut)); --- SQL operation complete. >>prepare s from insert overwrite table ins_store_sales_summary select >>ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales group by >>ss_sold_date_sk ,ss_store_sk; @@ -624,6 +637,10 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD --- SQL operation complete. >>execute s; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. + --- 12768 row(s) inserted. >>control query shape cut; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/EXPECTED005 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED005 b/core/sql/regress/hive/EXPECTED005 index 8e26184..eaa414e 100644 --- a/core/sql/regress/hive/EXPECTED005 +++ b/core/sql/regress/hive/EXPECTED005 @@ -161,11 +161,12 @@ Y 9525 >>insert into newtable values ('abc'); --- 1 row(s) inserted. ->>cqd query_cache '0'; - ---- SQL operation complete. >>select * from newtable; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. + A ------------------------- @@ -173,9 +174,6 @@ abc --- 1 row(s) selected. >>-- expect to see the row, but only because query cache is off ->>cqd query_cache reset; - ---- SQL operation complete. >> >>insert into hiveregr5.newtable2 values ('xyz'); @@ -289,9 +287,6 @@ A B >>-- overwrite the table with auto-generated partitions >>sh regrhive.ksh -v -f $REGRTSTDIR/TEST005_d.hive.sql; >> ->>cqd query_cache '0'; - ---- SQL operation complete. >>prepare s4 from +> select c_preferred_cust_flag, +> count(*) @@ -652,14 +647,18 @@ C1 C2 C3 C4 C >>sh echo "create table thive(a int);" > TEST005_junk; >>sh regrhive.ksh -f TEST005_junk; >> ->>select * from hive.hive.thive; +>>select a from hive.hive.thive; --- 0 row(s) selected. >> >>sh echo "insert into thive values (1);" > TEST005_junk; >>sh regrhive.ksh -f TEST005_junk; >> ->>select * from hive.hive.thive; +>>select a from hive.hive.thive; + +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. A ----------- @@ -670,7 +669,7 @@ A >>insert into hive.hive.thive values (2); --- 1 row(s) inserted. ->>select * from hive.hive.thive; +>>select a from hive.hive.thive; A ----------- @@ -689,6 +688,19 @@ A >>sh echo "insert into thive values (1,2);" > TEST005_junk; >>sh regrhive.ksh -f TEST005_junk; >> +>>select a from hive.hive.thive; + +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. + +A +----------- + + 1 + +--- 1 row(s) selected. +>> >>select * from hive.hive.thive; A B http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/EXPECTED015 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED015 b/core/sql/regress/hive/EXPECTED015 index f1d6067..162a3bc 100644 --- a/core/sql/regress/hive/EXPECTED015 +++ b/core/sql/regress/hive/EXPECTED015 @@ -115,10 +115,11 @@ >>cqd COMPRESSED_INTERNAL_FORMAT_DEFRAG_RATIO '100'; --- SQL operation complete. ->>cqd query_cache '0'; +>>cqd HIVE_NUM_ESPS_PER_DATANODE '3'; --- SQL operation complete. ->>cqd HIVE_NUM_ESPS_PER_DATANODE '3'; +>> +>>cqd auto_query_retry_warnings 'ON'; --- SQL operation complete. >> @@ -209,9 +210,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.T015T2 Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.T015T2 Task: PREPARATION Status: Started Object: TRAFODION.HBASE.T015T2 Rows Processed: 5 -Task: PREPARATION Status: Ended ET: 00:00:00.184 +Task: PREPARATION Status: Ended ET: 00:00:00.051 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.T015T2 -Task: COMPLETION Status: Ended ET: 00:00:00.359 +Task: COMPLETION Status: Ended ET: 00:00:00.781 --- 5 row(s) loaded. >> @@ -233,9 +234,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.T015T2 Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.T015T2 Task: PREPARATION Status: Started Object: TRAFODION.HBASE.T015T2 Rows Processed: 5 -Task: PREPARATION Status: Ended ET: 00:00:00.176 +Task: PREPARATION Status: Ended ET: 00:00:00.162 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.T015T2 -Task: COMPLETION Status: Ended ET: 00:00:00.203 +Task: COMPLETION Status: Ended ET: 00:00:00.183 --- 5 row(s) loaded. >> @@ -258,9 +259,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.T015T2 Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.T015T2 Task: PREPARATION Status: Started Object: TRAFODION.HBASE.T015T2 Rows Processed: 5 -Task: PREPARATION Status: Ended ET: 00:00:00.195 +Task: PREPARATION Status: Ended ET: 00:00:00.064 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.T015T2 -Task: COMPLETION Status: Ended ET: 00:00:00.318 +Task: COMPLETION Status: Ended ET: 00:00:00.327 --- 5 row(s) loaded. >> @@ -288,9 +289,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.T015T2 Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.T015T2 Task: PREPARATION Status: Started Object: TRAFODION.HBASE.T015T2 Rows Processed: 5 -Task: PREPARATION Status: Ended ET: 00:00:00.179 +Task: PREPARATION Status: Ended ET: 00:00:00.164 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.T015T2 -Task: COMPLETION Status: Ended ET: 00:00:00.380 +Task: COMPLETION Status: Ended ET: 00:00:00.268 --- 5 row(s) loaded. >> @@ -461,9 +462,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRE Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_ADDRESS Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS Rows Processed: 5000 -Task: PREPARATION Status: Ended ET: 00:00:08.673 +Task: PREPARATION Status: Ended ET: 00:00:08.885 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS -Task: COMPLETION Status: Ended ET: 00:00:00.402 +Task: COMPLETION Status: Ended ET: 00:00:01.379 --- 5000 row(s) loaded. >> @@ -543,9 +544,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRE Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK Rows Processed: 5000 -Task: PREPARATION Status: Ended ET: 00:00:01.062 +Task: PREPARATION Status: Ended ET: 00:00:01.054 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK -Task: COMPLETION Status: Ended ET: 00:00:00.294 +Task: COMPLETION Status: Ended ET: 00:00:00.820 --- 5000 row(s) loaded. >> @@ -644,9 +645,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS Rows Processed: 5000 -Task: PREPARATION Status: Ended ET: 00:00:04.130 +Task: PREPARATION Status: Ended ET: 00:00:03.594 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS -Task: COMPLETION Status: Ended ET: 00:00:00.631 +Task: COMPLETION Status: Ended ET: 00:00:00.323 --- 5000 row(s) loaded. >> @@ -691,8 +692,6 @@ CD_DEMO_SK CD_GENDER CD_MARITAL_STATUS CD_EDUCATION_STATUS CD_PURCHASE_EST >>--------------------- >>select count(*) from hive.hive.customer_demographics where cd_demo_sk <= >>5000; -*** WARNING[6008] Statistics for column (CD_DEMO_SK) from table HIVE.HIVE.CUSTOMER_DEMOGRAPHICS were not available. As a result, the access path chosen might not be the best possible. - (EXPR) -------------------- @@ -750,9 +749,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT Rows Processed: 5000 -Task: PREPARATION Status: Ended ET: 00:00:06.200 +Task: PREPARATION Status: Ended ET: 00:00:05.596 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT -Task: COMPLETION Status: Ended ET: 00:00:01.735 +Task: COMPLETION Status: Ended ET: 00:00:01.203 --- 5000 row(s) loaded. >> @@ -1262,9 +1261,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS Rows Processed: 5000 -Task: PREPARATION Status: Ended ET: 00:00:06.517 +Task: PREPARATION Status: Ended ET: 00:00:04.891 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS -Task: COMPLETION Status: Ended ET: 00:00:02.242 +Task: COMPLETION Status: Ended ET: 00:00:02.090 --- 5000 row(s) loaded. >> @@ -1305,11 +1304,11 @@ Task: DISABLE INDEXE Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG Task: DISABLE INDEXE Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS Rows Processed: 1000 -Task: PREPARATION Status: Ended ET: 00:00:04.908 +Task: PREPARATION Status: Ended ET: 00:00:04.588 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS -Task: COMPLETION Status: Ended ET: 00:00:01.357 +Task: COMPLETION Status: Ended ET: 00:00:01.075 Task: POPULATE INDEX Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS -Task: POPULATE INDEX Status: Ended ET: 00:00:12.289 +Task: POPULATE INDEX Status: Ended ET: 00:00:11.865 --- 1000 row(s) loaded. >> @@ -1354,11 +1353,11 @@ Task: DISABLE INDEXE Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG Task: DISABLE INDEXE Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT Rows Processed: 5000 -Task: PREPARATION Status: Ended ET: 00:00:05.592 +Task: PREPARATION Status: Ended ET: 00:00:05.812 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT -Task: COMPLETION Status: Ended ET: 00:00:02.018 +Task: COMPLETION Status: Ended ET: 00:00:02.106 Task: POPULATE INDEX Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT -Task: POPULATE INDEX Status: Ended ET: 00:00:14.563 +Task: POPULATE INDEX Status: Ended ET: 00:00:12.813 --- 5000 row(s) loaded. >> @@ -1532,9 +1531,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE."customer_addr Task: CLEANUP Status: Ended Object: TRAFODION.HBASE."customer_address_delim" Task: PREPARATION Status: Started Object: TRAFODION.HBASE."customer_address_delim" Rows Processed: 5000 -Task: PREPARATION Status: Ended ET: 00:00:04.154 +Task: PREPARATION Status: Ended ET: 00:00:03.585 Task: COMPLETION Status: Started Object: TRAFODION.HBASE."customer_address_delim" -Task: COMPLETION Status: Ended ET: 00:00:01.729 +Task: COMPLETION Status: Ended ET: 00:00:01.575 --- 5000 row(s) loaded. >> http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/TEST003 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST003 b/core/sql/regress/hive/TEST003 index 7e3641b..3761906 100644 --- a/core/sql/regress/hive/TEST003 +++ b/core/sql/regress/hive/TEST003 @@ -66,11 +66,12 @@ order by s desc; log LOG003 clear; set schema hive.hive; -cqd attempt_esp_parallelism 'off'; -cqd hive_max_esps '1'; -cqd PARALLEL_NUM_ESPS '1'; +--cqd attempt_esp_parallelism 'off'; +--cqd hive_max_esps '1'; +--cqd PARALLEL_NUM_ESPS '1'; cqd HIVE_MAX_STRING_LENGTH '25' ; cqd mode_seahive 'ON'; +cqd auto_query_retry_warnings 'ON'; prepare s from insert into hive.ins_customer select * from hive.customer; execute s; @@ -187,7 +188,7 @@ sh diff -b LOG003_ORIG_STORE_SALES_SUMMARY.DAT LOG003_INS_STORE_SALES_SUMMARY.DA log LOG003; --execute again --overwrite should get rid og existing data from previous run -control query shape union(cut,esp_exchange(cut)); +control query shape union(cut, esp_exchange(cut)); prepare s from insert overwrite table ins_store_sales_summary select ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales group by ss_sold_date_sk ,ss_store_sk; explain options 'f' s; execute s; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/TEST005 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST005 b/core/sql/regress/hive/TEST005 index e44b633..cd3fa56 100644 --- a/core/sql/regress/hive/TEST005 +++ b/core/sql/regress/hive/TEST005 @@ -144,10 +144,8 @@ where c_customer_sk between 20000 and 39999; select * from newtable; -- no rows, but should know the new table insert into newtable values ('abc'); -cqd query_cache '0'; select * from newtable; -- expect to see the row, but only because query cache is off -cqd query_cache reset; insert into hiveregr5.newtable2 values ('xyz'); select * from hiveregr5.newtable2; @@ -190,7 +188,6 @@ select a,b from newtable; -- overwrite the table with auto-generated partitions sh regrhive.ksh -v -f $REGRTSTDIR/TEST005_d.hive.sql; -cqd query_cache '0'; prepare s4 from select c_preferred_cust_flag, count(*) @@ -292,14 +289,14 @@ sh regrhive.ksh -f TEST005_junk; sh echo "create table thive(a int);" > TEST005_junk; sh regrhive.ksh -f TEST005_junk; -select * from hive.hive.thive; +select a from hive.hive.thive; sh echo "insert into thive values (1);" > TEST005_junk; sh regrhive.ksh -f TEST005_junk; -select * from hive.hive.thive; +select a from hive.hive.thive; insert into hive.hive.thive values (2); -select * from hive.hive.thive; +select a from hive.hive.thive; sh echo "drop table thive;" > TEST005_junk; sh regrhive.ksh -f TEST005_junk; @@ -310,6 +307,8 @@ sh regrhive.ksh -f TEST005_junk; sh echo "insert into thive values (1,2);" > TEST005_junk; sh regrhive.ksh -f TEST005_junk; +select a from hive.hive.thive; + select * from hive.hive.thive; log; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/TEST015 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST015 b/core/sql/regress/hive/TEST015 index ed0dd12..dfac48f 100644 --- a/core/sql/regress/hive/TEST015 +++ b/core/sql/regress/hive/TEST015 @@ -39,9 +39,10 @@ obey TEST015(setup); cqd COMPRESSED_INTERNAL_FORMAT 'ON'; cqd COMPRESSED_INTERNAL_FORMAT_BMO 'ON'; cqd COMPRESSED_INTERNAL_FORMAT_DEFRAG_RATIO '100'; -cqd query_cache '0'; cqd HIVE_NUM_ESPS_PER_DATANODE '3'; +cqd auto_query_retry_warnings 'ON'; + obey TEST015(test_bulk_load_simple); log;
