Repository: incubator-trafodion Updated Branches: refs/heads/master ea2087410 -> 249dfefac
Adding AQR retry for ENOENT and EAGAIN which may be encountered after an hdfsOpen on a file that has gone through a timestamp check. Squashed commit of the following: commit 52551b6461e0b761aeb37958ecd8673ad29bec89 Author: Sandhya Sundaresan <[email protected]> Date: Tue Jul 12 20:27:33 2016 +0000 Check for NULL pointer commit a7bb285fb7874b314acb6c74cde5e0b6911d6930 Author: Sandhya Sundaresan <[email protected]> Date: Tue Jul 12 16:12:17 2016 +0000 Rework to pass errno at the lowest layer as output param commit abb236336ee68765fcf9c534fbeae3fe6af5cdb6 Author: Sandhya Sundaresan <[email protected]> Date: Mon Jul 11 19:22:04 2016 +0000 Removed all retries and added AQR retry for missing hdfs file after timestamp check. commit 2530c4ebb8274bea8394067d2d2212b5bc3c97ef Author: Sandhya Sundaresan <[email protected]> Date: Mon Jul 11 07:11:35 2016 +0000 timeout to 20 commit 825bd5be2b0db4006bf82a0a9348338f7fc66236 Author: Sandhya Sundaresan <[email protected]> Date: Mon Jul 11 03:59:11 2016 +0000 increase sleep commit e471a4655ca8efe469e5dd8ab14748660fa5cdec Author: Sandhya Sundaresan <[email protected]> Date: Sun Jul 10 19:00:03 2016 +0000 Unconditionally retrying on open errors. commit b0402243a17094f7fec9899a6afc5c31c48cc38b Author: Sandhya Sundaresan <[email protected]> Date: Sun Jul 10 03:31:01 2016 +0000 Retrying for ENOENT too commit cbb2175282095edb73b30e79a9728f689b80c435 Author: Sandhya Sundaresan <[email protected]> Date: Sat Jul 9 23:14:31 2016 +0000 Rework and returning errno commit dd99c24deec09add76342e583193f79646fc9661 Author: Sandhya Sundaresan <[email protected]> Date: Sat Jul 9 05:05:43 2016 +0000 fixing build issue commit 66941774b1087ce564932af0c30ce50f15d0da67 Author: Sandhya Sundaresan <[email protected]> Date: Sat Jul 9 04:40:59 2016 +0000 Increasing retry and add hdfsClose to cleanup . commit 1934f2b3b7b32ee4a0396d9e56457ffc9f2cb9cf Author: Sandhya Sundaresan <[email protected]> Date: Fri Jul 8 21:59:00 2016 +0000 Adding retry logic when making hdfsOpenFile calls to read an hdfs file to handle potential EAGAIN errors. [TRAFODION-2097] Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/249dfefa Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/249dfefa Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/249dfefa Branch: refs/heads/master Commit: 249dfefac344fbb7db72d1131fae15a75c50058d Parents: ea20874 Author: Sandhya Sundaresan <[email protected]> Authored: Thu Jul 14 00:18:01 2016 +0000 Committer: Sandhya Sundaresan <[email protected]> Committed: Thu Jul 14 00:18:01 2016 +0000 ---------------------------------------------------------------------- core/sql/executor/ExHdfsScan.cpp | 28 ++++- core/sql/exp/ExpLOBaccess.cpp | 117 ++++++++++++-------- core/sql/exp/ExpLOBaccess.h | 2 +- core/sql/exp/ExpLOBinterface.cpp | 5 +- core/sql/exp/ExpLOBinterface.h | 3 +- core/sql/regress/hive/EXPECTED018 | 194 ++++++++++++++++----------------- core/sql/regress/hive/TEST018 | 24 ++-- 7 files changed, 207 insertions(+), 166 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/executor/ExHdfsScan.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp index 3b32d69..ec9f3f8 100644 --- a/core/sql/executor/ExHdfsScan.cpp +++ b/core/sql/executor/ExHdfsScan.cpp @@ -377,6 +377,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work() ContextCli *currContext = getGlobals()->castToExExeStmtGlobals()->getCliGlobals()->currContext(); hdfsFS hdfs = currContext->getHdfsServerConnection(hdfsScanTdb().hostName_,hdfsScanTdb().port_); hdfsFileInfo *dirInfo = NULL; + Int32 hdfsErrorDetail = 0;//this is errno returned form underlying hdfsOpenFile call. while (!qparent_.down->isEmpty()) { ex_queue_entry *pentry_down = qparent_.down->getHeadEntry(); @@ -561,6 +562,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work() } else { + openType = 2; // must open retcode = ExpLOBInterfaceSelectCursor (lobGlob_, @@ -582,9 +584,21 @@ ExWorkProcRetcode ExHdfsScanTcb::work() bytesRead_, NULL, 1, // open - openType // + openType, // + &hdfsErrorDetail ); + if ((retcode < 0) && + ((hdfsErrorDetail == ENOENT) || (hdfsErrorDetail == EAGAIN))) + { + ComDiagsArea * diagsArea = NULL; + ExRaiseSqlError(getHeap(), &diagsArea, + (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR)); + pentry_down->setDiagsArea(diagsArea); + step_ = HANDLE_ERROR_AND_DONE; + break; + } + // preopen next range. if ( (currRangeNum_ + 1) < (beginRangeNum_ + numRanges_) ) { @@ -615,7 +629,8 @@ ExWorkProcRetcode ExHdfsScanTcb::work() bytesRead_, NULL, 1,// open - openType + openType, + &hdfsErrorDetail ); hdfsFileName_ = hdfo_->fileName(); @@ -631,7 +646,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work() ExRaiseSqlError(getHeap(), &diagsArea, (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE), NULL, &intParam1, - &cliError, + &hdfsErrorDetail, NULL, "HDFS", (char*)"ExpLOBInterfaceSelectCursor/open", @@ -702,7 +717,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work() } else { - + Int32 hdfsErrorDetail = 0;///this is the errno returned from the underlying hdfs call. retcode = ExpLOBInterfaceSelectCursor (lobGlob_, hdfsFileName_, @@ -723,6 +738,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work() hdfsScanBuffer_ + trailingPrevRead_, 2, // read 0 // openType, not applicable for read + &hdfsErrorDetail ); if (hdfsStats_) @@ -737,7 +753,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work() ExRaiseSqlError(getHeap(), &diagsArea, (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE), NULL, &intParam1, - &cliError, + &hdfsErrorDetail, NULL, "HDFS", (char*)"ExpLOBInterfaceSelectCursor/read", @@ -1247,7 +1263,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work() ExRaiseSqlError(getHeap(), &diagsArea, (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE), NULL, &intParam1, - &cliError, + &errno, NULL, "HDFS", (char*)"ExpLOBInterfaceSelectCursor/close", http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/exp/ExpLOBaccess.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp index 67ea3be..575bb92 100644 --- a/core/sql/exp/ExpLOBaccess.cpp +++ b/core/sql/exp/ExpLOBaccess.cpp @@ -552,8 +552,9 @@ Ex_Lob_Error ExLob::statSourceFile(char *srcfile, Int64 &sourceEOF) if (srcType == HDFS_FILE) { hdfsFile sourceFile = hdfsOpenFile(fs_,srcfile,O_RDONLY,0,0,0); - if (!sourceFile) - return LOB_SOURCE_FILE_OPEN_ERROR; + if (!sourceFile) + return LOB_SOURCE_FILE_OPEN_ERROR; + hdfsFileInfo *sourceFileInfo = hdfsGetPathInfo(fs_,srcfile); // get EOD from source hdfs file. if (sourceFileInfo) @@ -669,11 +670,9 @@ Ex_Lob_Error ExLob::readHdfsSourceFile(char *srcfile, char *&fileData, Int32 &si int openFlags = O_RDONLY; hdfsFile fdSrcFile = hdfsOpenFile(fs_,srcfile, openFlags,0,0,0); - if (fdSrcFile == NULL) { + if (fdSrcFile == NULL) return LOB_SOURCE_FILE_OPEN_ERROR; - } - - + fileData = (char *) (getLobGlobalHeap())->allocateMemory(size); if (fileData == (char *)-1) { return LOB_SOURCE_DATA_ALLOC_ERROR; @@ -1301,8 +1300,11 @@ Ex_Lob_Error ExLob::openCursor(char *handleIn, Int32 handleInLen,Int64 transId) return LOB_OPER_OK; } -Ex_Lob_Error ExLob::openDataCursor(char *file, LobsCursorType type, Int64 range, Int64 bufMaxSize, - Int64 maxBytes, Int64 waited, ExLobGlobals *lobGlobals) +Ex_Lob_Error ExLob::openDataCursor(char *file, LobsCursorType type, + Int64 range, Int64 bufMaxSize, + Int64 maxBytes, Int64 waited, + ExLobGlobals *lobGlobals, + Int32 *hdfsDetailError) { Ex_Lob_Error err; cursor_t cursor; @@ -1359,22 +1361,27 @@ Ex_Lob_Error ExLob::openDataCursor(char *file, LobsCursorType type, Int64 range, it = lobCursors_.find(string(file, strlen(file))); // to get the actual cursor object in the map if (!fdData_ || (openFlags_ != O_RDONLY)) - { - hdfsCloseFile(fs_, fdData_); - fdData_ = NULL; - openFlags_ = O_RDONLY; - fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0); - if (!fdData_) { - openFlags_ = -1; - lobCursorLock_.unlock(); - return LOB_DATA_FILE_OPEN_ERROR; + { + hdfsCloseFile(fs_, fdData_); + fdData_ = NULL; + openFlags_ = O_RDONLY; + fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0); + + if (!fdData_) + { + openFlags_ = -1; + if (hdfsDetailError) + *hdfsDetailError = errno; + lobCursorLock_.unlock(); + return LOB_DATA_FILE_OPEN_ERROR; + } + + if (hdfsSeek(fs_, fdData_, (it->second).descOffset_) == -1) + { + lobCursorLock_.unlock(); + return LOB_DATA_FILE_POSITION_ERROR; + } } - } - - if (hdfsSeek(fs_, fdData_, (it->second).descOffset_) == -1) { - lobCursorLock_.unlock(); - return LOB_DATA_FILE_POSITION_ERROR; - } // start reading in a worker thread lobGlobals->enqueuePrefetchRequest(this, &(it->second)); @@ -1552,15 +1559,17 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int hdfsFile fdData = hdfsOpenFile(fs, lobDataFile_, O_RDONLY, 0, 0,0); - if (!fdData) - { + + if (!fdData) + { str_sprintf(logBuf,"Could not open file:%s",lobDataFile_); lobDebugInfo(logBuf,0,__LINE__,lobTrace_); hdfsCloseFile(fs,fdData); fdData = NULL; return LOB_DATA_FILE_OPEN_ERROR; } - + + hdfsFile fdTemp = hdfsOpenFile(fs, tmpLobDataFile,O_WRONLY|O_CREAT,0,0,0); if (!fdTemp) { @@ -1768,16 +1777,20 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I // #endif if (!fdData_ || (openFlags_ != O_RDONLY)) - { - hdfsCloseFile(fs_, fdData_); - fdData_=NULL; - openFlags_ = O_RDONLY; - fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0); - if (!fdData_) { - openFlags_ = -1; - return LOB_DATA_FILE_OPEN_ERROR; - } - } + { + hdfsCloseFile(fs_, fdData_); + fdData_=NULL; + openFlags_ = O_RDONLY; + fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0); + + if (!fdData_) + { + openFlags_ = -1; + return LOB_DATA_FILE_OPEN_ERROR; + } + + + } clock_gettime(CLOCK_MONOTONIC, &startTime); @@ -1844,20 +1857,30 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr, fdData_=NULL; openFlags_ = O_RDONLY; fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0); - if (!fdData_) { - openFlags_ = -1; - return LOB_DATA_FILE_OPEN_ERROR; - } + + if (!fdData_) + { + openFlags_ = -1; + return LOB_DATA_FILE_OPEN_ERROR; + } + + } else { fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0); - if (!fdData_) { - openFlags_ = -1; - return LOB_DATA_FILE_OPEN_ERROR; - } + + if (!fdData_) + { + openFlags_ = -1; + return LOB_DATA_FILE_OPEN_ERROR; + } + + } - + + + if (!multipleChunks) { lobDebugInfo("Reading in single chunk",0,__LINE__,lobTrace_); @@ -2361,7 +2384,7 @@ Ex_Lob_Error ExLobsOper ( } else if (openType == 2) { // must open sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId); fileName = fn; - err = lobPtr->openDataCursor(fileName, Lob_Cursor_Simple, descNumIn, sourceLen, cursorBytes, waited, lobGlobals); + err = lobPtr->openDataCursor(fileName, Lob_Cursor_Simple, descNumIn, sourceLen, cursorBytes, waited, lobGlobals, (Int32 *)blackBox); } else err = LOB_SUBOPER_ERROR; break; @@ -3306,7 +3329,7 @@ Ex_Lob_Error ExLobGlobals::processPreOpens() lobPtr->openDataCursor(preOpenObj->cursorName_, Lob_Cursor_Simple, preOpenObj->range_, preOpenObj->bufMaxSize_, preOpenObj->maxBytes_, - preOpenObj->waited_, this); + preOpenObj->waited_, this,0); } return LOB_OPER_OK; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/exp/ExpLOBaccess.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h index c8be6a9..b05eb8f 100644 --- a/core/sql/exp/ExpLOBaccess.h +++ b/core/sql/exp/ExpLOBaccess.h @@ -429,7 +429,7 @@ class ExLob bool hasNoOpenCursors() { return lobCursors_.empty(); } Ex_Lob_Error openCursor(char *handleIn, Int32 handleInLen,Int64 transId); Ex_Lob_Error openDataCursor(char *fileName, LobsCursorType type, Int64 range, - Int64 bytesLeft, Int64 bufMaxSize, Int64 prefetch, ExLobGlobals *lobGlobals); + Int64 bytesLeft, Int64 bufMaxSize, Int64 prefetch, ExLobGlobals *lobGlobals, Int32 *hdfsDetailError = NULL); Ex_Lob_Error deleteCursor(char *cursorName, ExLobGlobals *lobGlobals); Ex_Lob_Error fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOffset, Int64 &outSize,NABoolean &isEOD,Int64 transId); Ex_Lob_Error insertData(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize,char *handleIn,Int32 handleInLen, char *blackBox, Int32 blackBoxLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/exp/ExpLOBinterface.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBinterface.cpp b/core/sql/exp/ExpLOBinterface.cpp index 56f7e90..9c3c213 100644 --- a/core/sql/exp/ExpLOBinterface.cpp +++ b/core/sql/exp/ExpLOBinterface.cpp @@ -884,7 +884,8 @@ Lng32 ExpLOBInterfaceSelectCursor(void * exLobGlob, Int64 &outLen, char * lobData, Lng32 oper, // 1: open. 2: fetch. 3: close - Lng32 openType // 0: not applicable. 1: preOpen. 2: mustOpen. + Lng32 openType, // 0: not applicable. 1: preOpen. 2: mustOpen. + Int32 *hdfsDetailError ) { Ex_Lob_Error err; @@ -942,7 +943,7 @@ Lng32 ExpLOBInterfaceSelectCursor(void * exLobGlob, waitedOp, exLobGlob, 0, - NULL, 0,0,0,0,0,0,0, + hdfsDetailError, 0,0,0,0,0,0,0, openType ); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/exp/ExpLOBinterface.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBinterface.h b/core/sql/exp/ExpLOBinterface.h index ed50169..cb6fda8 100644 --- a/core/sql/exp/ExpLOBinterface.h +++ b/core/sql/exp/ExpLOBinterface.h @@ -274,7 +274,8 @@ Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob, Int64 &outLen, char * lobData, Lng32 oper, // 1: open. 2: fetch. 3: close - Lng32 openType // 0: not applicable. 1: preOpen. 2: mustOpen. + Lng32 openType, // 0: not applicable. 1: preOpen. 2: mustOpen. + Int32 *hdfsDetailError = NULL ); Lng32 ExpLOBinterfaceStats(void * lobGlob, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/regress/hive/EXPECTED018 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED018 b/core/sql/regress/hive/EXPECTED018 index 4228108..a0dedea 100644 --- a/core/sql/regress/hive/EXPECTED018 +++ b/core/sql/regress/hive/EXPECTED018 @@ -150,9 +150,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRE Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_ADDRESS Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS Rows Processed: 50000 -Task: PREPARATION Status: Ended ET: 00:00:10.156 +Task: PREPARATION Status: Ended ET: 00:00:09.733 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS -Task: COMPLETION Status: Ended ET: 00:00:00.404 +Task: COMPLETION Status: Ended ET: 00:00:00.398 --- 50000 row(s) loaded. >>-- @@ -181,9 +181,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS Rows Processed: 20000 -Task: PREPARATION Status: Ended ET: 00:00:11.022 +Task: PREPARATION Status: Ended ET: 00:00:12.160 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS -Task: COMPLETION Status: Ended ET: 00:00:00.279 +Task: COMPLETION Status: Ended ET: 00:00:00.362 --- 20000 row(s) loaded. >>-- @@ -213,9 +213,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT Rows Processed: 20000 -Task: PREPARATION Status: Ended ET: 00:00:08.435 +Task: PREPARATION Status: Ended ET: 00:00:08.208 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT -Task: COMPLETION Status: Ended ET: 00:00:00.266 +Task: COMPLETION Status: Ended ET: 00:00:00.327 --- 20000 row(s) loaded. >>-- >> @@ -235,9 +235,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_SALT Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT Rows Processed: 100000 -Task: PREPARATION Status: Ended ET: 00:00:10.885 +Task: PREPARATION Status: Ended ET: 00:00:10.404 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT -Task: COMPLETION Status: Ended ET: 00:00:00.258 +Task: COMPLETION Status: Ended ET: 00:00:00.332 --- 100000 row(s) loaded. >>-- @@ -266,9 +266,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.STORE_SALES_SA Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.STORE_SALES_SALT Task: PREPARATION Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT Rows Processed: 160756 -Task: PREPARATION Status: Ended ET: 00:00:13.092 +Task: PREPARATION Status: Ended ET: 00:00:13.314 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT -Task: COMPLETION Status: Ended ET: 00:00:00.843 +Task: COMPLETION Status: Ended ET: 00:00:00.309 --- 160756 row(s) loaded. >>-- @@ -356,10 +356,10 @@ a +> select * from null_format_src; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: EXTRACT Status: Started Rows Processed: 10 -Task: EXTRACT Status: Ended ET: 00:00:00.284 +Task: EXTRACT Status: Ended ET: 00:00:00.334 --- 10 row(s) unloaded. >>select * from hive.hive.null_format_default; @@ -385,10 +385,10 @@ a +> select * from null_format_src; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.008 Task: EXTRACT Status: Started Rows Processed: 10 -Task: EXTRACT Status: Ended ET: 00:00:00.366 +Task: EXTRACT Status: Ended ET: 00:00:00.325 --- 10 row(s) unloaded. >>select * from hive.hive.null_format_empty; @@ -417,7 +417,7 @@ Task: EMPTY TARGET Status: Started Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 10 -Task: EXTRACT Status: Ended ET: 00:00:00.293 +Task: EXTRACT Status: Ended ET: 00:00:00.409 --- 10 row(s) unloaded. >>select * from hive.hive.null_format_colon; @@ -471,12 +471,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.011 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:03.450 +Task: EXTRACT Status: Ended ET: 00:00:04.200 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.042 +Task: MERGE FILES Status: Ended ET: 00:00:00.064 --- 50000 row(s) unloaded. >>log; @@ -509,12 +509,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.976 +Task: EXTRACT Status: Ended ET: 00:00:01.146 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.034 +Task: MERGE FILES Status: Ended ET: 00:00:00.039 --- 20000 row(s) unloaded. >>log; @@ -533,12 +533,12 @@ cat /tmp/merged_customer_demogs | wc -l +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:01.204 +Task: EXTRACT Status: Ended ET: 00:00:00.983 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.031 +Task: MERGE FILES Status: Ended ET: 00:00:00.024 --- 20000 row(s) unloaded. >>log; @@ -570,12 +570,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.010 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:01.108 +Task: EXTRACT Status: Ended ET: 00:00:01.386 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.040 +Task: MERGE FILES Status: Ended ET: 00:00:00.052 --- 20000 row(s) unloaded. >> @@ -596,9 +596,9 @@ Task: EMPTY TARGET Status: Started Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.873 +Task: EXTRACT Status: Ended ET: 00:00:00.911 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.039 +Task: MERGE FILES Status: Ended ET: 00:00:00.049 --- 20000 row(s) unloaded. >> @@ -631,10 +631,10 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.007 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.803 +Task: EXTRACT Status: Ended ET: 00:00:01.244 --- 20000 row(s) unloaded. >> @@ -654,12 +654,12 @@ regrhadoop.ksh fs -ls /bulkload/customer_demographics_salt/file* | grep file | +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.011 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.010 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.838 +Task: EXTRACT Status: Ended ET: 00:00:01.058 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.046 +Task: MERGE FILES Status: Ended ET: 00:00:00.057 --- 20000 row(s) unloaded. >> @@ -792,12 +792,12 @@ CD_DEMO_SK CD_GENDER +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.837 +Task: EXTRACT Status: Ended ET: 00:00:00.982 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.042 +Task: MERGE FILES Status: Ended ET: 00:00:00.048 --- 20000 row(s) unloaded. >>log; @@ -832,15 +832,15 @@ regrhadoop.ksh fs -ls /bulkload/customer_demographics_salt/merged* | grep merge +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.007 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.836 +Task: EXTRACT Status: Ended ET: 00:00:01.064 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.040 +Task: MERGE FILES Status: Ended ET: 00:00:00.059 --- 20000 row(s) unloaded. ->>sh sleep 10; +>>--sh sleep 10; >>select count(*) from hive.hive.unload_customer_demographics; (EXPR) @@ -887,13 +887,13 @@ CD_DEMO_SK CD_GENDER +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.946 +Task: EXTRACT Status: Ended ET: 00:00:00.995 --- 20000 row(s) unloaded. ->>sh sleep 10; +>>--sh sleep 10; >>select count(*) from hive.hive.unload_customer_demographics; *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. @@ -941,13 +941,13 @@ CD_DEMO_SK CD_GENDER +>select * from trafodion.hbase.customer_address ; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.007 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:02.325 +Task: EXTRACT Status: Ended ET: 00:00:02.475 --- 50000 row(s) unloaded. ->>sh sleep 10; +>>--sh sleep 10; >>select count(*) from hive.hive.unload_customer_address; (EXPR) @@ -997,10 +997,10 @@ Task: EMPTY TARGET Status: Started Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:02.438 +Task: EXTRACT Status: Ended ET: 00:00:02.595 --- 50000 row(s) unloaded. ->>sh sleep 10; +>>--sh sleep 10; >>select count(*) from hive.hive.unload_customer_address; *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. @@ -1063,10 +1063,10 @@ Task: EMPTY TARGET Status: Started Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 Task: EXTRACT Status: Started Rows Processed: 100000 -Task: EXTRACT Status: Ended ET: 00:00:07.083 +Task: EXTRACT Status: Ended ET: 00:00:07.642 --- 100000 row(s) unloaded. ->>sh sleep 10; +>>--sh sleep 10; >>select count(*) from hive.hive.unload_customer; (EXPR) @@ -1114,13 +1114,13 @@ C_CUSTOMER_SK C_CUSTOMER_ID +>select * from trafodion.hbase.customer_demographics_salt; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.010 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.019 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:01.226 +Task: EXTRACT Status: Ended ET: 00:00:01.228 --- 20000 row(s) unloaded. ->>sh sleep 10; +>>--sh sleep 10; >>select count(*) from hive.hive.unload_customer_demographics; (EXPR) @@ -1167,12 +1167,12 @@ CD_DEMO_SK CD_GENDER +>select * from trafodion.hbase.customer_address where ca_address_sk < 100; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.003 Task: EXTRACT Status: Started Rows Processed: 99 -Task: EXTRACT Status: Ended ET: 00:00:00.244 +Task: EXTRACT Status: Ended ET: 00:00:00.216 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.024 +Task: MERGE FILES Status: Ended ET: 00:00:00.023 --- 99 row(s) unloaded. >> @@ -1211,10 +1211,10 @@ Task: EMPTY TARGET Status: Started Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: EXTRACT Status: Started Rows Processed: 12349 -Task: EXTRACT Status: Ended ET: 00:00:07.776 +Task: EXTRACT Status: Ended ET: 00:00:10.062 --- 12349 row(s) unloaded. ->>sh sleep 10; +>>--sh sleep 10; >>select [first 100] * from hive.hive.unload_store_sales_summary order by >>ss_sold_date_sk,ss_store_sk; SS_SOLD_DATE_SK SS_STORE_SK SS_QUANTITY @@ -1333,10 +1333,10 @@ Task: EMPTY TARGET Status: Started Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 100000 -Task: EXTRACT Status: Ended ET: 00:00:08.464 +Task: EXTRACT Status: Ended ET: 00:00:09.838 --- 100000 row(s) unloaded. ->>sh sleep 10; +>>--sh sleep 10; >>select count(*) from hive.hive.unload_customer_and_address; (EXPR) @@ -1381,13 +1381,13 @@ C_CUSTOMER_SK C_CUSTOMER_ID +>select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.008 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: EXTRACT Status: Started Rows Processed: 1998 Task: EXTRACT Status: Ended ET: 00:00:00.695 --- 1998 row(s) unloaded. ->>sh sleep 10; +>>--sh sleep 10; >>select count(*) from hive.hive.unload_customer_address; (EXPR) @@ -1495,7 +1495,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2 use_snapshot_scan ...... TRUE full_table_name ........ TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT snapshot_name .......... TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT_SNAP111 - snapshot_temp_location /bulkload/20160706204232/ + snapshot_temp_location /bulkload/20160708212424/ grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id >>--no snapshot >>explain reg; @@ -1575,7 +1575,7 @@ grep -i -e 'explain snp' -e snapshot -e full_table_name -e esp_exchange LOG018_S use_snapshot_scan ...... TRUE full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111 - snapshot_temp_location /bulkload/20160706204243/ + snapshot_temp_location /bulkload/20160708212436/ grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id >>--no snapshot >>explain reg; @@ -1657,11 +1657,11 @@ grep -i -e 'explain snp' -e snapshot -e full_table_name -e esp_exchange LOG018_S use_snapshot_scan ...... TRUE full_table_name ........ TRAFODION.HBASE.CUSTOMER_SALT snapshot_name .......... TRAFODION.HBASE.CUSTOMER_SALT_SNAP111 - snapshot_temp_location /bulkload/20160706204314/ + snapshot_temp_location /bulkload/20160708212512/ use_snapshot_scan ...... TRUE full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111 - snapshot_temp_location /bulkload/20160706204314/ + snapshot_temp_location /bulkload/20160708212512/ grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id >>--no snapshot >>explain reg; @@ -1774,13 +1774,13 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.025 Task: VERIFY SNAPSHO Status: Started Snapshots verified: 1 -Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.425 +Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.494 Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:02.515 +Task: EXTRACT Status: Ended ET: 00:00:03.271 --- 50000 row(s) unloaded. >> @@ -1852,13 +1852,13 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.003 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.002 Task: VERIFY SNAPSHO Status: Started Snapshots verified: 1 -Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.333 +Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.374 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:02.008 +Task: EXTRACT Status: Ended ET: 00:00:02.347 --- 20000 row(s) unloaded. >> @@ -1906,16 +1906,16 @@ CD_DEMO_SK CD_GENDER +>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.016 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.008 Task: CREATE SNAPSHO Status: Started Snapshots created: 1 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.707 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.520 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:01.166 +Task: EXTRACT Status: Ended ET: 00:00:01.725 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 1 -Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.026 +Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.028 --- 20000 row(s) unloaded. >> @@ -1970,13 +1970,13 @@ Task: EMPTY TARGET Status: Started Task: EMPTY TARGET Status: Ended ET: 00:00:00.009 Task: CREATE SNAPSHO Status: Started Snapshots created: 1 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.515 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.458 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:01.263 +Task: EXTRACT Status: Ended ET: 00:00:01.347 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 1 -Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.019 +Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.006 --- 20000 row(s) unloaded. >> @@ -2029,16 +2029,16 @@ CD_DEMO_SK CD_GENDER +>select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 Task: CREATE SNAPSHO Status: Started Snapshots created: 1 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.233 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.079 Task: EXTRACT Status: Started Rows Processed: 1998 -Task: EXTRACT Status: Ended ET: 00:00:00.847 +Task: EXTRACT Status: Ended ET: 00:00:00.725 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 1 -Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.006 +Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.005 --- 1998 row(s) unloaded. >> @@ -2113,19 +2113,19 @@ CA_ADDRESS_SK CA_ADDRESS_ID +>select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.048 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: CREATE SNAPSHO Status: Started Snapshots created: 2 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:02.709 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:03.077 Task: EXTRACT Status: Started Rows Processed: 100000 -Task: EXTRACT Status: Ended ET: 00:00:09.569 +Task: EXTRACT Status: Ended ET: 00:00:09.442 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 2 -Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.010 +Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.009 --- 100000 row(s) unloaded. ->>sh sleep 10; +>>--sh sleep 10; >>select count(*) from hive.hive.unload_customer_and_address; (EXPR) @@ -2190,19 +2190,19 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +>select c_first_name,c_last_name from trafodion.hbase.customer_salt; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: CREATE SNAPSHO Status: Started Snapshots created: 1 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.410 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.640 Task: EXTRACT Status: Started Rows Processed: 100000 -Task: EXTRACT Status: Ended ET: 00:00:01.589 +Task: EXTRACT Status: Ended ET: 00:00:01.597 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 1 -Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.006 +Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.007 --- 100000 row(s) unloaded. ->>sh sleep 10; +>>--sh sleep 10; >>select count(*) from hive.hive.unload_customer_name; (EXPR) @@ -2272,7 +2272,7 @@ unload with delimiter 0 into '/bulkload/test' select * from CUSTOMER_ADDRESS; Task: UNLOAD Status: Started Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:02.253 +Task: EXTRACT Status: Ended ET: 00:00:02.827 --- 50000 row(s) unloaded. >>--unload 24 -- should give an error @@ -2337,10 +2337,10 @@ regrhadoop.ksh fs -rm /user/hive/exttables/unload_customer_demographics/* Task: UNLOAD Status: Started Task: EXTRACT Status: Started Rows Processed but NOT Written to Disk: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.840 +Task: EXTRACT Status: Ended ET: 00:00:00.904 --- 20000 row(s) unloaded. ->>sh sleep 10; +>>--sh sleep 10; >>select count(*) from hive.hive.unload_customer_demographics; (EXPR) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/regress/hive/TEST018 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST018 b/core/sql/regress/hive/TEST018 index 0932da5..5f1b611 100644 --- a/core/sql/regress/hive/TEST018 +++ b/core/sql/regress/hive/TEST018 @@ -446,7 +446,7 @@ MERGE FILE 'merged_customer_demographics' OVERWRITE INTO '/user/hive/exttables/unload_customer_demographics' select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>; -sh sleep 10; +--sh sleep 10; select count(*) from hive.hive.unload_customer_demographics; select [first 20] * from hive.hive.unload_customer_demographics where cd_demo_sk < 100 order by cd_demo_sk; @@ -458,7 +458,7 @@ WITH PURGEDATA FROM TARGET INTO '/user/hive/exttables/unload_customer_demographics' select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>; -sh sleep 10; +--sh sleep 10; select count(*) from hive.hive.unload_customer_demographics; select [first 20] * from hive.hive.unload_customer_demographics where cd_demo_sk <100 order by cd_demo_sk; @@ -467,7 +467,7 @@ UNLOAD WITH PURGEDATA FROM TARGET INTO '/user/hive/exttables/unload_customer_address' select * from trafodion.hbase.customer_address ; -sh sleep 10; +--sh sleep 10; select count(*) from hive.hive.unload_customer_address; select [first 20] * from hive.hive.unload_customer_address where ca_address_sk <100 order by ca_address_sk; @@ -477,7 +477,7 @@ UNLOAD WITH PURGEDATA FROM TARGET DELIMITER 124 RECORD_SEPARATOR 10 INTO '/user/hive/exttables/unload_customer_address' select * from trafodion.hbase.customer_address ; -sh sleep 10; +--sh sleep 10; select count(*) from hive.hive.unload_customer_address; select [first 20] * from hive.hive.unload_customer_address where ca_address_sk < 100 order by ca_address_sk; @@ -492,7 +492,7 @@ WITH PURGEDATA FROM TARGET --COMPRESSION GZIP INTO '/user/hive/exttables/unload_customer' select * from trafodion.hbase.customer_salt; -sh sleep 10; +--sh sleep 10; select count(*) from hive.hive.unload_customer; select [first 20] * from hive.hive.unload_customer where c_customer_sk < 100 order by c_customer_sk; @@ -503,7 +503,7 @@ WITH PURGEDATA FROM TARGET --COMPRESSION GZIP INTO '/user/hive/exttables/unload_customer_demographics' select * from trafodion.hbase.customer_demographics_salt; -sh sleep 10; +--sh sleep 10; select count(*) from hive.hive.unload_customer_demographics; select [first 20] * from hive.hive.unload_customer_demographics where cd_demo_sk < 100 order by cd_demo_sk; @@ -540,7 +540,7 @@ WITH PURGEDATA FROM TARGET DELIMITER '|' RECORD_SEPARATOR '\n' INTO '/user/hive/exttables/unload_store_sales_summary' select ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales_salt group by ss_sold_date_sk ,ss_store_sk; -sh sleep 10; +--sh sleep 10; select [first 100] * from hive.hive.unload_store_sales_summary order by ss_sold_date_sk,ss_store_sk; --unload 18 @@ -548,7 +548,7 @@ UNLOAD WITH PURGEDATA FROM TARGET INTO '/user/hive/exttables/unload_customer_and_address' select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ; -sh sleep 10; +--sh sleep 10; select count(*) from hive.hive.unload_customer_and_address; select [first 20] * from hive.hive.unload_customer_and_address order by ca_address_sk,c_customer_sk; @@ -558,7 +558,7 @@ WITH PURGEDATA FROM TARGET INTO '/user/hive/exttables/unload_customer_address' select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000; -sh sleep 10; +--sh sleep 10; select count(*) from hive.hive.unload_customer_address; select [first 20] * from hive.hive.unload_customer_address order by ca_address_sk; select [first 20] * from hive.hive.unload_customer_address order by ca_address_sk desc; @@ -695,7 +695,7 @@ WITH PURGEDATA FROM TARGET NEW SNAPSHOT HAVING SUFFIX 'SNAP' INTO '/user/hive/exttables/unload_customer_and_address' select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ; -sh sleep 10; +--sh sleep 10; select count(*) from hive.hive.unload_customer_and_address; select [first 20] * from hive.hive.unload_customer_and_address order by ca_address_sk,c_customer_sk; @@ -711,7 +711,7 @@ WITH PURGEDATA FROM TARGET NEW SNAPSHOT HAVING SUFFIX 'SNAP111' INTO '/user/hive/exttables/unload_customer_name' select c_first_name,c_last_name from trafodion.hbase.customer_salt; -sh sleep 10; +--sh sleep 10; select count(*) from hive.hive.unload_customer_name; select [first 20] * from hive.hive.unload_customer_name order by c_first_name,c_last_name; @@ -753,7 +753,7 @@ UNLOAD WITH PURGEDATA FROM TARGET INTO '/user/hive/exttables/unload_customer_demographics' (select * from trafodion.hbase.customer_demographics_salt) ; -sh sleep 10; +--sh sleep 10; select count(*) from hive.hive.unload_customer_demographics; CQD TRAF_UNLOAD_SKIP_WRITING_TO_FILES reset;
