hive data modification detection: commit #3
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/b1a8f024 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/b1a8f024 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/b1a8f024 Branch: refs/heads/master Commit: b1a8f024b5a3d4db368b51bbd2b6aac7035b9571 Parents: fa70e68 Author: Anoop Sharma <anoop.sha...@esgyn.com> Authored: Mon May 30 13:48:16 2016 +0000 Committer: Anoop Sharma <anoop.sha...@esgyn.com> Committed: Mon May 30 13:48:16 2016 +0000 ---------------------------------------------------------------------- core/sql/comexe/ComTdbFastTransport.cpp | 3 +- core/sql/comexe/ComTdbFastTransport.h | 5 +- core/sql/executor/ExFastTransport.cpp | 87 +++++++++-- core/sql/executor/ExFastTransport.h | 5 + core/sql/exp/ExpLOBaccess.cpp | 155 +++++++++++++++++--- core/sql/exp/ExpLOBaccess.h | 4 +- core/sql/generator/GenFastTransport.cpp | 60 +++++--- core/sql/optimizer/HDFSHook.cpp | 3 +- core/sql/regress/hive/EXPECTED003 | 9 -- core/sql/regress/hive/EXPECTED005 | 8 +- core/sql/regress/hive/EXPECTED006 | 3 - core/sql/regress/hive/EXPECTED018 | 208 +++++++++++++++------------ core/sql/regress/hive/TEST003 | 3 - core/sql/regress/hive/TEST006 | 1 - core/sql/regress/hive/TEST018 | 4 +- 15 files changed, 386 insertions(+), 172 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/comexe/ComTdbFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbFastTransport.cpp b/core/sql/comexe/ComTdbFastTransport.cpp index d995fdb..f492e8f 100644 --- a/core/sql/comexe/ComTdbFastTransport.cpp +++ b/core/sql/comexe/ComTdbFastTransport.cpp @@ -98,7 +98,8 @@ ComTdbFastExtract::ComTdbFastExtract( hdfsIOBufferSize_(hdfBuffSize), hdfsReplication_(replication), ioTimeout_(ioTimeout), - childDataRowLen_(childDataRowLen) + childDataRowLen_(childDataRowLen), + modTSforDir_(-1) { } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/comexe/ComTdbFastTransport.h ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbFastTransport.h b/core/sql/comexe/ComTdbFastTransport.h index 1ae7625..98e93eb 100644 --- a/core/sql/comexe/ComTdbFastTransport.h +++ b/core/sql/comexe/ComTdbFastTransport.h @@ -401,6 +401,8 @@ public: return childDataRowLen_; } + void setModTSforDir(Int64 v) { modTSforDir_ = v; } + Int64 getModTSforDir() const { return modTSforDir_; } protected: NABasicPtr targetName_; // 00 - 07 @@ -427,9 +429,10 @@ protected: UInt16 ioTimeout_; // 128 - 129 UInt16 filler_; // 130 - 131 UInt32 childDataRowLen_; // 132 - 135 + Int64 modTSforDir_; // 136 - 143 // Make sure class size is a multiple of 8 - char fillerComTdbFastTransport_[8]; // 136 - 143 + char fillerComTdbFastTransport_[8]; // 144 - 151 }; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/executor/ExFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExFastTransport.cpp b/core/sql/executor/ExFastTransport.cpp index b881b14..45c3959 100644 --- a/core/sql/executor/ExFastTransport.cpp +++ b/core/sql/executor/ExFastTransport.cpp @@ -89,6 +89,7 @@ ExFastExtractTcb::ExFastExtractTcb( , sourceFieldsConvIndex_(NULL) , currBuffer_(NULL) , bufferAllocFailuresCount_(0) + , modTS_(-1) { ex_globals *stmtGlobals = getGlobals(); @@ -482,6 +483,16 @@ Lng32 ExHdfsFastExtractTcb::lobInterfaceCreate() } +Lng32 ExHdfsFastExtractTcb::lobInterfaceDataModCheck() +{ + return ExpLOBinterfaceDataModCheck(lobGlob_, + targetLocation_, + hdfsHost_, + hdfsPort_, + myTdb().getModTSforDir(), + 0); +} + Lng32 ExHdfsFastExtractTcb::lobInterfaceClose() { @@ -530,13 +541,13 @@ Int32 ExHdfsFastExtractTcb::fixup() ex_tcb::fixup(); - if(!myTdb().getSkipWritingToFiles() && !myTdb().getBypassLibhdfs()) ExpLOBinterfaceInit (lobGlob_, getGlobals()->getDefaultHeap(),TRUE); + modTS_ = myTdb().getModTSforDir(); return 0; } @@ -681,9 +692,62 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work() { case EXTRACT_NOT_STARTED: { + pstate.step_= EXTRACT_CHECK_MOD_TS; + } + break; + + case EXTRACT_CHECK_MOD_TS: + { + if ((! myTdb().getTargetFile()) || + (myTdb().getModTSforDir() == -1)) + { + pstate.step_ = EXTRACT_INITIALIZE; + break; + } + + numBuffers_ = 0; + + memset (hdfsHost_, '\0', sizeof(hdfsHost_)); + strncpy(hdfsHost_, myTdb().getHdfsHostName(), sizeof(hdfsHost_)); + hdfsPort_ = myTdb().getHdfsPortNum(); + memset (fileName_, '\0', sizeof(fileName_)); + memset (targetLocation_, '\0', sizeof(targetLocation_)); + snprintf(targetLocation_,999, "%s", myTdb().getTargetName()); + + retcode = lobInterfaceDataModCheck(); + if (retcode < 0) + { + Lng32 cliError = 0; + + Lng32 intParam1 = -retcode; + ComDiagsArea * diagsArea = NULL; + ExRaiseSqlError(getHeap(), &diagsArea, + (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE), + NULL, &intParam1, + &cliError, + NULL, + "HDFS", + (char*)"ExpLOBInterfaceDataModCheck", + getLobErrStr(intParam1)); + pentry_down->setDiagsArea(diagsArea); + pstate.step_ = EXTRACT_ERROR; + break; + } + + if (retcode == 1) // check failed + { + ComDiagsArea * diagsArea = NULL; + ExRaiseSqlError(getHeap(), &diagsArea, + (ExeErrorCode)(8436)); + pentry_down->setDiagsArea(diagsArea); + pstate.step_ = EXTRACT_ERROR; + break; + } + pstate.step_= EXTRACT_INITIALIZE; } - // no break here + break; + case EXTRACT_INITIALIZE: { pstate.processingStarted_ = FALSE; @@ -798,7 +862,7 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work() break; } } - + if (feStats) { feStats->setPartitionNumber(fileNum); @@ -1123,13 +1187,16 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work() } else if (myTdb().getBypassLibhdfs()) { - sfwRetCode = sequenceFileWriter_->hdfsClose(); - if (!errorOccurred_ && sfwRetCode != SFW_OK ) - { - createSequenceFileError(sfwRetCode); - pstate.step_ = EXTRACT_ERROR; - break; - } + if (sequenceFileWriter_) + { + sfwRetCode = sequenceFileWriter_->hdfsClose(); + if (!errorOccurred_ && sfwRetCode != SFW_OK ) + { + createSequenceFileError(sfwRetCode); + pstate.step_ = EXTRACT_ERROR; + break; + } + } } else { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/executor/ExFastTransport.h ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExFastTransport.h b/core/sql/executor/ExFastTransport.h index 2df9fe9..5de05aa 100644 --- a/core/sql/executor/ExFastTransport.h +++ b/core/sql/executor/ExFastTransport.h @@ -255,6 +255,7 @@ public: enum FastExtractStates { EXTRACT_NOT_STARTED = 0, + EXTRACT_CHECK_MOD_TS, EXTRACT_INITIALIZE, EXTRACT_PASS_REQUEST_TO_CHILD, EXTRACT_RETURN_ROWS_FROM_CHILD, @@ -366,6 +367,9 @@ protected: time_t tstart_; UInt32 bufferAllocFailuresCount_; + + // modification timestamp of root dir location. + Int64 modTS_; }; // class ExFastExtractTcb ///////////////////////////////////////////////////// @@ -401,6 +405,7 @@ protected: Lng32 lobInterfaceInsert(ssize_t bytesToWrite); Lng32 lobInterfaceCreate(); Lng32 lobInterfaceClose(); + Lng32 lobInterfaceDataModCheck(); virtual void insertUpQueueEntry(ex_queue::up_status status, ComDiagsArea *diags, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/exp/ExpLOBaccess.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp index 3ac537b..7db4a40 100644 --- a/core/sql/exp/ExpLOBaccess.cpp +++ b/core/sql/exp/ExpLOBaccess.cpp @@ -110,8 +110,7 @@ ExLob::~ExLob() } -__thread hdfsFS *globalFS = NULL; - +#ifdef __ignore Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode, char *dir, LobsStorage storage, @@ -147,20 +146,13 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode, hdfsServer_ = hdfsServer; hdfsPort_ = hdfsPort; - if (globalFS == NULL) - { - globalFS = new hdfsFS; - *globalFS = NULL; - } - - if (*globalFS == NULL) + if (fs_ == NULL) { - *globalFS = hdfsConnect(hdfsServer_, hdfsPort_); - if (*globalFS == NULL) + fs_ = hdfsConnect(hdfsServer_, hdfsPort_); + if (fs_ == NULL) return LOB_HDFS_CONNECT_ERROR; } - fs_ = *globalFS; if (lobGlobals) lobGlobals->setHdfsFs(fs_); @@ -214,6 +206,103 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode, return LOB_OPER_OK; } +#endif + +Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode, + char *dir, + LobsStorage storage, + char *hdfsServer, Int64 hdfsPort, + char *lobLocation, + int bufferSize , short replication , + int blockSize, Int64 lobMaxSize, ExLobGlobals *lobGlobals) +{ + int openFlags; + mode_t filePerms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; + struct timespec startTime; + struct timespec endTime; + Int64 secs, nsecs, totalnsecs; + + if (dir) + { + if (dir_.empty()) + { + dir_ = string(dir); + } + + if (lobFile) + snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s/%s", dir_.c_str(), + lobFile); + + } + else + { + if (lobFile) + snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s", lobFile); + + } + + if (storage_ != Lob_Invalid_Storage) + { + return LOB_INIT_ERROR; + } else + { + storage_ = storage; + } + + stats_.init(); + + hdfsServer_ = hdfsServer; + hdfsPort_ = hdfsPort; + lobLocation_ = lobLocation; + clock_gettime(CLOCK_MONOTONIC, &startTime); + + if (lobGlobals->getHdfsFs() == NULL) + { + fs_ = hdfsConnect(hdfsServer_, hdfsPort_); + if (fs_ == NULL) + return LOB_HDFS_CONNECT_ERROR; + lobGlobals->setHdfsFs(fs_); + } + else + { + fs_ = lobGlobals->getHdfsFs(); + } + + clock_gettime(CLOCK_MONOTONIC, &endTime); + + secs = endTime.tv_sec - startTime.tv_sec; + nsecs = endTime.tv_nsec - startTime.tv_nsec; + if (nsecs < 0) + { + secs--; + nsecs += NUM_NSECS_IN_SEC; + } + totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs; + stats_.hdfsConnectionTime += totalnsecs; + + if (mode == EX_LOB_CREATE) + { + // check if file is already created + hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_); + if (fInfo != NULL) + { + hdfsFreeFileInfo(fInfo, 1); + return LOB_DATA_FILE_CREATE_ERROR; + } + openFlags = O_WRONLY | O_CREAT; + fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags, bufferSize, replication, blockSize); + if (!fdData_) + { + return LOB_DATA_FILE_CREATE_ERROR; + } + hdfsCloseFile(fs_, fdData_); + fdData_ = NULL; + + } + lobGlobalHeap_ = lobGlobals->getHeap(); + return LOB_OPER_OK; + +} Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOffset, Int64 &outSize,NABoolean &isEOD, Int64 transId) { @@ -443,13 +532,24 @@ Ex_Lob_Error ExLob::dataModCheck2( Ex_Lob_Error ExLob::dataModCheck( char * dirPath, Int64 inputModTS, - Lng32 numOfPartLevels) + Lng32 numOfPartLevels, + ExLobGlobals *lobGlobals) { // find mod time of root dir hdfsFileInfo *fileInfos = hdfsGetPathInfo(fs_, dirPath); if (fileInfos == NULL) { - return LOB_DATA_FILE_NOT_FOUND_ERROR; + hdfsDisconnect(fs_); + fs_ = hdfsConnect(hdfsServer_, hdfsPort_); + if (fs_ == NULL) + return LOB_HDFS_CONNECT_ERROR; + + fileInfos = hdfsGetPathInfo(fs_, dirPath); + if (fileInfos == NULL) + return LOB_DIR_NAME_ERROR; + + if (lobGlobals) + lobGlobals->setHdfsFs(fs_); } Int64 currModTS = fileInfos[0].mLastMod; @@ -471,12 +571,18 @@ Ex_Lob_Error ExLob::emptyDirectory() Ex_Lob_Error err; int numExistingFiles=0; - hdfsFileInfo *fileInfos = hdfsListDirectory(fs_, lobDataFile_, &numExistingFiles); + hdfsFileInfo *fileInfos = hdfsGetPathInfo(fs_, lobDataFile_); if (fileInfos == NULL) - { - return LOB_DATA_FILE_NOT_FOUND_ERROR; //here a directory - } + { + return LOB_DATA_FILE_NOT_FOUND_ERROR; //here a directory + } + fileInfos = hdfsListDirectory(fs_, lobDataFile_, &numExistingFiles); + if (fileInfos == NULL) + { + return LOB_OPER_OK; + } + for (int i = 0; i < numExistingFiles; i++) { #ifdef USE_HADOOP_1 @@ -2167,7 +2273,8 @@ Ex_Lob_Error ExLobsOper ( if (globPtr == NULL) { - if (operation == Lob_Init) + if ((operation == Lob_Init) || + (operation == Lob_Data_Mod_Check)) { globPtr = (void *) new ExLobGlobals(); if (globPtr == NULL) @@ -2176,14 +2283,16 @@ Ex_Lob_Error ExLobsOper ( lobGlobals = (ExLobGlobals *)globPtr; err = lobGlobals->initialize(); - return err; + if (err != LOB_OPER_OK) + return err; } else { return LOB_GLOB_PTR_ERROR; } } - else + + if ((globPtr != NULL) && (operation != Lob_Init)) { lobGlobals = (ExLobGlobals *)globPtr; @@ -2236,6 +2345,7 @@ Ex_Lob_Error ExLobsOper ( */ switch(operation) { + case Lob_Init: case Lob_Create: break; @@ -2425,7 +2535,8 @@ Ex_Lob_Error ExLobsOper ( Int64 inputModTS = *(Int64*)blackBox; Int32 inputNumOfPartLevels = *(Lng32*)&((char*)blackBox)[sizeof(inputModTS)]; - err = lobPtr->dataModCheck(dir, inputModTS, inputNumOfPartLevels); + err = lobPtr->dataModCheck(dir, inputModTS, inputNumOfPartLevels, + lobGlobals); } break; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/exp/ExpLOBaccess.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h index 518fbd7..416529d 100644 --- a/core/sql/exp/ExpLOBaccess.h +++ b/core/sql/exp/ExpLOBaccess.h @@ -489,7 +489,9 @@ class ExLob Ex_Lob_Error dataModCheck( char * dirPath, Int64 modTS, - Lng32 numOfPartLevels); + Lng32 numOfPartLevels, + ExLobGlobals *lobGlobals); + Ex_Lob_Error dataModCheck2( char * dirPath, Int64 modTS, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/generator/GenFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenFastTransport.cpp b/core/sql/generator/GenFastTransport.cpp index 953176b..7e1ee3e 100644 --- a/core/sql/generator/GenFastTransport.cpp +++ b/core/sql/generator/GenFastTransport.cpp @@ -562,6 +562,18 @@ PhysicalFastExtract::codeGen(Generator *generator) newRecordSep[1] = '\0'; } + Int64 modTS = -1; + if ((CmpCommon::getDefault(HIVE_DATA_MOD_CHECK) == DF_ON) && + (isHiveInsert()) && + (getHiveTableDesc() && getHiveTableDesc()->getNATable() && + getHiveTableDesc()->getNATable()->getClusteringIndex())) + { + const HHDFSTableStats* hTabStats = + getHiveTableDesc()->getNATable()->getClusteringIndex()->getHHDFSTableStats(); + + modTS = hTabStats->getModificationTS(); + } + targetName = AllocStringInSpace(*space, (char *)getTargetName().data()); hdfsHostName = AllocStringInSpace(*space, (char *)getHdfsHostName().data()); hiveTableName = AllocStringInSpace(*space, (char *)getHiveTableName().data()); @@ -570,27 +582,27 @@ PhysicalFastExtract::codeGen(Generator *generator) recordSeparator = AllocStringInSpace(*space, newRecordSep); nullString = AllocStringInSpace(*space, (char *)getNullString().data()); - result = ft_codegen(generator, - *this, // RelExpr &relExpr - newTdb, // ComTdbUdr *&newTdb - estimatedRowCount, - targetName, - hdfsHostName, - hdfsPortNum, - hiveTableName, - delimiter, - header, - nullString, - recordSeparator, - downQueueMaxSize, - upQueueMaxSize, - outputBufferSize, - requestBufferSize, - replyBufferSize, - numOutputBuffers, - childTdb, - isSequenceFile()); - + result = ft_codegen(generator, + *this, // RelExpr &relExpr + newTdb, // ComTdbUdr *&newTdb + estimatedRowCount, + targetName, + hdfsHostName, + hdfsPortNum, + hiveTableName, + delimiter, + header, + nullString, + recordSeparator, + downQueueMaxSize, + upQueueMaxSize, + outputBufferSize, + requestBufferSize, + replyBufferSize, + numOutputBuffers, + childTdb, + isSequenceFile()); + if (!generator->explainDisabled()) { generator->setExplainTuple(addExplainInfo(newTdb, firstExplainTuple, 0, generator)); @@ -626,8 +638,10 @@ PhysicalFastExtract::codeGen(Generator *generator) else GenAssert(0, "Unexpected Fast Extract compression type") } - if((ActiveSchemaDB()->getDefaults()).getToken(FAST_EXTRACT_DIAGS) == DF_ON) - newTdb->setPrintDiags(1); + if((ActiveSchemaDB()->getDefaults()).getToken(FAST_EXTRACT_DIAGS) == DF_ON) + newTdb->setPrintDiags(1); + + newTdb->setModTSforDir(modTS); return result; } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/optimizer/HDFSHook.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/HDFSHook.cpp b/core/sql/optimizer/HDFSHook.cpp index 4d3f9b6..fda6611 100644 --- a/core/sql/optimizer/HDFSHook.cpp +++ b/core/sql/optimizer/HDFSHook.cpp @@ -1070,7 +1070,8 @@ void HHDFSTableStats::print(FILE *ofd) fprintf(ofd,"====================================================================\n"); } -extern __thread hdfsFS *globalFS; +//extern __thread hdfsFS *globalFS; +hdfsFS *globalFS; NABoolean HHDFSTableStats::connectHDFS(const NAString &host, Int32 port) { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/EXPECTED003 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED003 b/core/sql/regress/hive/EXPECTED003 index 46aede2..79cdb5a 100644 --- a/core/sql/regress/hive/EXPECTED003 +++ b/core/sql/regress/hive/EXPECTED003 @@ -5,9 +5,6 @@ >>cqd attempt_esp_parallelism 'off'; --- SQL operation complete. ->>cqd AUTO_QUERY_RETRY 'OFF'; - ---- SQL operation complete. >>cqd hive_max_esps '1'; --- SQL operation complete. @@ -152,9 +149,6 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S >> >>--try new HIVE SYNTAX >>-------------- ->>cqd query_cache '0'; - ---- SQL operation complete. >>insert into TABLE hive.ins_customer_address select * from >>hive.customer_address; --- 50000 row(s) inserted. @@ -529,9 +523,6 @@ T_TIME_SK T_TIME_ID T_TIME T_HOUR T_MINUTE T >>cqd HIVE_MAX_STRING_LENGTH '25' ; --- SQL operation complete. ->>cqd query_cache '0'; - ---- SQL operation complete. >>control query shape esp_exchange(cut); --- SQL operation complete. http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/EXPECTED005 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED005 b/core/sql/regress/hive/EXPECTED005 index 3c286d6..8e26184 100644 --- a/core/sql/regress/hive/EXPECTED005 +++ b/core/sql/regress/hive/EXPECTED005 @@ -226,9 +226,9 @@ xyz C_PREFERRED_CUST_FLAG (EXPR) ------------------------- -------------------- - 19631 + 1384 +N 19631 Y 18984 -? 1384 --- 3 row(s) selected. >>execute s3; @@ -316,9 +316,9 @@ A B C_PREFERRED_CUST_FLAG (EXPR) ------------------------- -------------------- - 19631 + 1384 +N 19631 Y 18984 -? 1384 --- 3 row(s) selected. >>execute s4; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/EXPECTED006 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED006 b/core/sql/regress/hive/EXPECTED006 index 645cca9..87f630e 100644 --- a/core/sql/regress/hive/EXPECTED006 +++ b/core/sql/regress/hive/EXPECTED006 @@ -4,9 +4,6 @@ --- SQL operation complete. >>set terminal_charset utf8; >> ->>cqd AUTO_QUERY_RETRY 'OFF'; - ---- SQL operation complete. >>cqd HIVE_MAX_STRING_LENGTH '25' ; --- SQL operation complete. http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/EXPECTED018 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED018 b/core/sql/regress/hive/EXPECTED018 index 01f4e0b..2c34c31 100644 --- a/core/sql/regress/hive/EXPECTED018 +++ b/core/sql/regress/hive/EXPECTED018 @@ -150,9 +150,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRE Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_ADDRESS Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS Rows Processed: 50000 -Task: PREPARATION Status: Ended ET: 00:00:09.173 +Task: PREPARATION Status: Ended ET: 00:00:06.864 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS -Task: COMPLETION Status: Ended ET: 00:00:00.304 +Task: COMPLETION Status: Ended ET: 00:00:00.262 --- 50000 row(s) loaded. >>-- @@ -181,9 +181,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS Rows Processed: 20000 -Task: PREPARATION Status: Ended ET: 00:00:11.448 +Task: PREPARATION Status: Ended ET: 00:00:10.758 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS -Task: COMPLETION Status: Ended ET: 00:00:00.275 +Task: COMPLETION Status: Ended ET: 00:00:00.234 --- 20000 row(s) loaded. >>-- @@ -213,9 +213,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT Rows Processed: 20000 -Task: PREPARATION Status: Ended ET: 00:00:07.319 +Task: PREPARATION Status: Ended ET: 00:00:09.668 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT -Task: COMPLETION Status: Ended ET: 00:00:00.198 +Task: COMPLETION Status: Ended ET: 00:00:00.208 --- 20000 row(s) loaded. >>-- >> @@ -235,9 +235,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_SALT Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT Rows Processed: 100000 -Task: PREPARATION Status: Ended ET: 00:00:08.340 +Task: PREPARATION Status: Ended ET: 00:00:07.549 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT -Task: COMPLETION Status: Ended ET: 00:00:00.212 +Task: COMPLETION Status: Ended ET: 00:00:00.167 --- 100000 row(s) loaded. >>-- @@ -266,9 +266,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.STORE_SALES_SA Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.STORE_SALES_SALT Task: PREPARATION Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT Rows Processed: 160756 -Task: PREPARATION Status: Ended ET: 00:00:10.675 +Task: PREPARATION Status: Ended ET: 00:00:11.382 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT -Task: COMPLETION Status: Ended ET: 00:00:00.256 +Task: COMPLETION Status: Ended ET: 00:00:00.180 --- 160756 row(s) loaded. >>-- @@ -288,6 +288,10 @@ Task: COMPLETION Status: Ended ET: 00:00:00.256 >>-- using insert >>insert overwrite table hive.hive.null_format_default select * from >>null_format_src; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. + --- 10 row(s) inserted. >>select * from hive.hive.null_format_default; @@ -309,6 +313,10 @@ a >> >>insert overwrite table hive.hive.null_format_empty select * from >>null_format_src; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. + --- 10 row(s) inserted. >>select * from hive.hive.null_format_empty; @@ -330,6 +338,10 @@ a ? >> >>insert overwrite table hive.hive.null_format_colon select * from >>null_format_src; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. + --- 10 row(s) inserted. >>select * from hive.hive.null_format_colon; @@ -356,10 +368,10 @@ a +> select * from null_format_src; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 10 -Task: EXTRACT Status: Ended ET: 00:00:00.238 +Task: EXTRACT Status: Ended ET: 00:00:00.232 --- 10 row(s) unloaded. >>select * from hive.hive.null_format_default; @@ -385,10 +397,10 @@ a +> select * from null_format_src; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 10 -Task: EXTRACT Status: Ended ET: 00:00:00.246 +Task: EXTRACT Status: Ended ET: 00:00:00.218 --- 10 row(s) unloaded. >>select * from hive.hive.null_format_empty; @@ -414,10 +426,10 @@ a ? +> select * from null_format_src; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.009 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 Task: EXTRACT Status: Started Rows Processed: 10 -Task: EXTRACT Status: Ended ET: 00:00:00.222 +Task: EXTRACT Status: Ended ET: 00:00:00.213 --- 10 row(s) unloaded. >>select * from hive.hive.null_format_colon; @@ -471,12 +483,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.011 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.007 Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:02.786 +Task: EXTRACT Status: Ended ET: 00:00:03.805 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.036 +Task: MERGE FILES Status: Ended ET: 00:00:00.034 --- 50000 row(s) unloaded. >>log; @@ -509,12 +521,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.010 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.868 +Task: EXTRACT Status: Ended ET: 00:00:00.865 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.031 +Task: MERGE FILES Status: Ended ET: 00:00:00.023 --- 20000 row(s) unloaded. >>log; @@ -533,12 +545,12 @@ cat /tmp/merged_customer_demogs | wc -l +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.008 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.837 +Task: EXTRACT Status: Ended ET: 00:00:00.766 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.029 +Task: MERGE FILES Status: Ended ET: 00:00:00.023 --- 20000 row(s) unloaded. >>log; @@ -570,12 +582,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.007 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:01.024 +Task: EXTRACT Status: Ended ET: 00:00:00.855 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.041 +Task: MERGE FILES Status: Ended ET: 00:00:00.032 --- 20000 row(s) unloaded. >> @@ -593,12 +605,12 @@ regrhadoop.ksh fs -du -s /bulkload/customer_demographics_salt/merged_customer_de +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.722 +Task: EXTRACT Status: Ended ET: 00:00:00.692 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.048 +Task: MERGE FILES Status: Ended ET: 00:00:00.031 --- 20000 row(s) unloaded. >> @@ -631,10 +643,10 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.013 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.591 +Task: EXTRACT Status: Ended ET: 00:00:00.484 --- 20000 row(s) unloaded. >> @@ -654,12 +666,12 @@ regrhadoop.ksh fs -ls /bulkload/customer_demographics_salt/file* | grep file | +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.012 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.010 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.672 +Task: EXTRACT Status: Ended ET: 00:00:00.580 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.040 +Task: MERGE FILES Status: Ended ET: 00:00:00.038 --- 20000 row(s) unloaded. >> @@ -792,12 +804,12 @@ CD_DEMO_SK CD_GENDER +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.616 +Task: EXTRACT Status: Ended ET: 00:00:00.487 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.055 +Task: MERGE FILES Status: Ended ET: 00:00:00.030 --- 20000 row(s) unloaded. >>log; @@ -835,9 +847,9 @@ Task: EMPTY TARGET Status: Started Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.562 +Task: EXTRACT Status: Ended ET: 00:00:00.492 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.046 +Task: MERGE FILES Status: Ended ET: 00:00:00.033 --- 20000 row(s) unloaded. >> @@ -887,15 +899,19 @@ CD_DEMO_SK CD_GENDER +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.480 +Task: EXTRACT Status: Ended ET: 00:00:00.980 --- 20000 row(s) unloaded. >> >>select count(*) from hive.hive.unload_customer_demographics; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. + (EXPR) -------------------- @@ -937,10 +953,10 @@ CD_DEMO_SK CD_GENDER +>select * from trafodion.hbase.customer_address ; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.007 Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:01.946 +Task: EXTRACT Status: Ended ET: 00:00:02.111 --- 50000 row(s) unloaded. >> @@ -990,15 +1006,19 @@ CA_ADDRESS_SK CA_ADDRESS_ID +>select * from trafodion.hbase.customer_address ; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.003 Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:01.891 +Task: EXTRACT Status: Ended ET: 00:00:02.075 --- 50000 row(s) unloaded. >> >>select count(*) from hive.hive.unload_customer_address; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. + (EXPR) -------------------- @@ -1052,10 +1072,10 @@ CA_ADDRESS_SK CA_ADDRESS_ID +>select * from trafodion.hbase.customer_salt; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 100000 -Task: EXTRACT Status: Ended ET: 00:00:06.124 +Task: EXTRACT Status: Ended ET: 00:00:06.402 --- 100000 row(s) unloaded. >>select count(*) from hive.hive.unload_customer; @@ -1105,10 +1125,10 @@ C_CUSTOMER_SK C_CUSTOMER_ID +>select * from trafodion.hbase.customer_demographics_salt; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.144 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.008 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.930 +Task: EXTRACT Status: Ended ET: 00:00:00.985 --- 20000 row(s) unloaded. >> @@ -1158,12 +1178,12 @@ CD_DEMO_SK CD_GENDER +>select * from trafodion.hbase.customer_address where ca_address_sk < 100; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: EXTRACT Status: Started Rows Processed: 99 -Task: EXTRACT Status: Ended ET: 00:00:00.207 +Task: EXTRACT Status: Ended ET: 00:00:00.203 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.029 +Task: MERGE FILES Status: Ended ET: 00:00:00.021 --- 99 row(s) unloaded. >> @@ -1199,10 +1219,10 @@ regrhadoop.ksh fs -rm /user/hive/exttables/unload_customer_demographics/* +>select ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales_salt group by ss_sold_date_sk ,ss_store_sk; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 Task: EXTRACT Status: Started Rows Processed: 12349 -Task: EXTRACT Status: Ended ET: 00:00:06.278 +Task: EXTRACT Status: Ended ET: 00:00:06.351 --- 12349 row(s) unloaded. >> @@ -1321,10 +1341,10 @@ SS_SOLD_DATE_SK SS_STORE_SK SS_QUANTITY +>select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 100000 -Task: EXTRACT Status: Ended ET: 00:00:07.279 +Task: EXTRACT Status: Ended ET: 00:00:07.648 --- 100000 row(s) unloaded. >> @@ -1372,10 +1392,10 @@ C_CUSTOMER_SK C_CUSTOMER_ID +>select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.141 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.008 Task: EXTRACT Status: Started Rows Processed: 1998 -Task: EXTRACT Status: Ended ET: 00:00:00.957 +Task: EXTRACT Status: Ended ET: 00:00:00.490 --- 1998 row(s) unloaded. >> @@ -1486,7 +1506,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2 use_snapshot_scan ...... TRUE full_table_name ........ TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT snapshot_name .......... TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT_SNAP111 - snapshot_temp_location /bulkload/20160526204449/ + snapshot_temp_location /bulkload/20160530055515/ grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id >>--no snapshot >>explain reg; @@ -1566,7 +1586,7 @@ grep -i -e 'explain snp' -e snapshot -e full_table_name -e esp_exchange LOG018_S use_snapshot_scan ...... TRUE full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111 - snapshot_temp_location /bulkload/20160526204456/ + snapshot_temp_location /bulkload/20160530055522/ grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id >>--no snapshot >>explain reg; @@ -1648,11 +1668,11 @@ grep -i -e 'explain snp' -e snapshot -e full_table_name -e esp_exchange LOG018_S use_snapshot_scan ...... TRUE full_table_name ........ TRAFODION.HBASE.CUSTOMER_SALT snapshot_name .......... TRAFODION.HBASE.CUSTOMER_SALT_SNAP111 - snapshot_temp_location /bulkload/20160526204518/ + snapshot_temp_location /bulkload/20160530055545/ use_snapshot_scan ...... TRUE full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111 - snapshot_temp_location /bulkload/20160526204518/ + snapshot_temp_location /bulkload/20160530055545/ grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id >>--no snapshot >>explain reg; @@ -1765,13 +1785,13 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 Task: VERIFY SNAPSHO Status: Started Snapshots verified: 1 -Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.288 +Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.294 Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:02.176 +Task: EXTRACT Status: Ended ET: 00:00:02.144 --- 50000 row(s) unloaded. >> @@ -1843,13 +1863,13 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.002 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.003 Task: VERIFY SNAPSHO Status: Started Snapshots verified: 1 -Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.337 +Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.311 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:01.975 +Task: EXTRACT Status: Ended ET: 00:00:02.081 --- 20000 row(s) unloaded. >> @@ -1900,18 +1920,22 @@ Task: EMPTY TARGET Status: Started Task: EMPTY TARGET Status: Ended ET: 00:00:00.010 Task: CREATE SNAPSHO Status: Started Snapshots created: 1 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.474 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.416 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:01.272 +Task: EXTRACT Status: Ended ET: 00:00:01.191 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 1 -Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.014 +Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.010 --- 20000 row(s) unloaded. >> >>select count(*) from hive.hive.unload_customer_demographics; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. + (EXPR) -------------------- @@ -1954,16 +1978,16 @@ CD_DEMO_SK CD_GENDER +>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.020 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.009 Task: CREATE SNAPSHO Status: Started Snapshots created: 1 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.597 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.602 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:01.095 +Task: EXTRACT Status: Ended ET: 00:00:01.481 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 1 -Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.006 +Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.005 --- 20000 row(s) unloaded. >> @@ -1977,6 +2001,10 @@ Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.006 --- 1 row(s) selected. >>select [first 20] * from hive.hive.unload_customer_demographics where >>cd_demo_sk < 100 order by cd_demo_sk; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. + CD_DEMO_SK CD_GENDER CD_MARITAL_STATUS CD_EDUCATION_STATUS CD_PURCHASE_ESTIMATE CD_CREDIT_RATING CD_DEP_COUNT CD_DEP_EMPLOYED_COUNT CD_DEP_COLLEGE_COUNT ----------- ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- -------------------- ---------------------------------------------------------------------------------------------------- ------------ --------------------- -------------------- @@ -2012,16 +2040,16 @@ CD_DEMO_SK CD_GENDER +>select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: CREATE SNAPSHO Status: Started Snapshots created: 1 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.048 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.627 Task: EXTRACT Status: Started Rows Processed: 1998 -Task: EXTRACT Status: Ended ET: 00:00:00.662 +Task: EXTRACT Status: Ended ET: 00:00:00.648 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 1 -Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.003 +Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.004 --- 1998 row(s) unloaded. >> @@ -2096,13 +2124,13 @@ CA_ADDRESS_SK CA_ADDRESS_ID +>select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.007 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: CREATE SNAPSHO Status: Started Snapshots created: 2 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:02.782 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:02.377 Task: EXTRACT Status: Started Rows Processed: 100000 -Task: EXTRACT Status: Ended ET: 00:00:08.942 +Task: EXTRACT Status: Ended ET: 00:00:08.169 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 2 Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.007 @@ -2173,16 +2201,16 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +>select c_first_name,c_last_name from trafodion.hbase.customer_salt; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.008 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.007 Task: CREATE SNAPSHO Status: Started Snapshots created: 1 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.365 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.420 Task: EXTRACT Status: Started Rows Processed: 100000 -Task: EXTRACT Status: Ended ET: 00:00:01.183 +Task: EXTRACT Status: Ended ET: 00:00:01.144 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 1 -Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.003 +Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.004 --- 100000 row(s) unloaded. >> @@ -2255,7 +2283,7 @@ unload with delimiter 0 into '/bulkload/test' select * from CUSTOMER_ADDRESS; Task: UNLOAD Status: Started Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:01.997 +Task: EXTRACT Status: Ended ET: 00:00:01.974 --- 50000 row(s) unloaded. >>--unload 24 -- should give an error @@ -2320,7 +2348,7 @@ regrhadoop.ksh fs -rm /user/hive/exttables/unload_customer_demographics/* Task: UNLOAD Status: Started Task: EXTRACT Status: Started Rows Processed but NOT Written to Disk: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.709 +Task: EXTRACT Status: Ended ET: 00:00:00.690 --- 20000 row(s) unloaded. >>select count(*) from hive.hive.unload_customer_demographics; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/TEST003 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST003 b/core/sql/regress/hive/TEST003 index ada9c87..7e3641b 100644 --- a/core/sql/regress/hive/TEST003 +++ b/core/sql/regress/hive/TEST003 @@ -67,7 +67,6 @@ log LOG003 clear; set schema hive.hive; cqd attempt_esp_parallelism 'off'; -cqd AUTO_QUERY_RETRY 'OFF'; cqd hive_max_esps '1'; cqd PARALLEL_NUM_ESPS '1'; cqd HIVE_MAX_STRING_LENGTH '25' ; @@ -101,7 +100,6 @@ insert into hive.ins_time_dim values ('a', 2, 3, 4, 5, 6, 'c', 'd', 'e', 'f'); --try new HIVE SYNTAX -------------- -cqd query_cache '0'; insert into TABLE hive.ins_customer_address select * from hive.customer_address; select count(*) from hive.customer_address; @@ -168,7 +166,6 @@ cqd PARALLEL_NUM_ESPS '2'; set schema hive; cqd hive_max_esps '2'; cqd HIVE_MAX_STRING_LENGTH '25' ; -cqd query_cache '0'; control query shape esp_exchange(cut); prepare s from insert into table ins_store_sales_summary select ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales group by ss_sold_date_sk ,ss_store_sk; explain options 'f' s; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/TEST006 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST006 b/core/sql/regress/hive/TEST006 index 10e3d55..37a2c8b 100644 --- a/core/sql/regress/hive/TEST006 +++ b/core/sql/regress/hive/TEST006 @@ -33,7 +33,6 @@ log LOG006 clear; set schema hive.hive; set terminal_charset utf8; -cqd AUTO_QUERY_RETRY 'OFF'; cqd HIVE_MAX_STRING_LENGTH '25' ; cqd HIST_ROWCOUNT_REQUIRING_STATS '50000'; cqd mode_seahive 'ON'; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/TEST018 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST018 b/core/sql/regress/hive/TEST018 index 034a2fb..86758e6 100644 --- a/core/sql/regress/hive/TEST018 +++ b/core/sql/regress/hive/TEST018 @@ -27,13 +27,11 @@ set schema trafodion.hbase; cqd comp_bool_226 'on'; cqd TRAF_TABLE_SNAPSHOT_SCAN_TABLE_SIZE_THRESHOLD '0'; cqd hive_max_string_length '60'; -cqd query_cache '0'; cqd HIST_ROWCOUNT_REQUIRING_STATS '50000'; +cqd AUTO_QUERY_RETRY_WARNINGS 'ON'; obey TEST018(clean_up); - - log LOG018 clear; sh regrhive.ksh -v -f $REGRTSTDIR/TEST018_create_hive_tables.hive &> $REGRRUNDIR/LOG018_create_hive_tables.log ;