Repository: incubator-trafodion Updated Branches: refs/heads/master 5071a2088 -> c7199d5aa
Fix for AQR for missing hive tables and misc lob related fixes TRAFODION-2770 Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/17ba6b8a Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/17ba6b8a Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/17ba6b8a Branch: refs/heads/master Commit: 17ba6b8acb69a3feb3cfd3c67ae6abc37d30b242 Parents: f49b86f Author: Sandhya Sundaresan <[email protected]> Authored: Mon Oct 16 17:54:18 2017 +0000 Committer: Sandhya Sundaresan <[email protected]> Committed: Mon Oct 16 17:54:18 2017 +0000 ---------------------------------------------------------------------- core/sql/cli/SessionDefaults.cpp | 2 +- core/sql/cli/Statement.cpp | 19 ++++++++--- core/sql/executor/ExExeUtilLoad.cpp | 3 +- core/sql/executor/ExHdfsScan.cpp | 10 ++++-- core/sql/exp/ExpLOBaccess.cpp | 21 ++++++++++-- core/sql/optimizer/ItemFunc.h | 2 +- core/sql/regress/hive/DIFF003.KNOWN | 56 +++++++++++++++++++++++++++----- core/sql/regress/hive/FILTER005 | 2 ++ 8 files changed, 96 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/cli/SessionDefaults.cpp ---------------------------------------------------------------------- diff --git a/core/sql/cli/SessionDefaults.cpp b/core/sql/cli/SessionDefaults.cpp index 25dd67c..966ce77 100644 --- a/core/sql/cli/SessionDefaults.cpp +++ b/core/sql/cli/SessionDefaults.cpp @@ -824,7 +824,7 @@ static const AQRInfo::AQRErrorMap aqrErrorMap[] = AQREntry( 8575, 0, 1, 0, 1, 0, "", 0, 0), // table not found - AQREntry( 8577, 0, 1, 0, 1, 0, "", 0, 0), + AQREntry( 8577, 0, 1, 0, 1, 2, "04:05", 0, 0), // sim check failure AQREntry( 8579, 0, 1, 0, 1, 0, "", 0, 0), http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/cli/Statement.cpp ---------------------------------------------------------------------- diff --git a/core/sql/cli/Statement.cpp b/core/sql/cli/Statement.cpp index 2169373..52f7e5a 100644 --- a/core/sql/cli/Statement.cpp +++ b/core/sql/cli/Statement.cpp @@ -2051,11 +2051,22 @@ RETCODE Statement::doHiveTableSimCheck(TrafSimilarityTableInfo *si, << DgString2(getLobErrStr(intParam1)) << DgInt0(intParam1) << DgInt1(0); - - if (intParam1 == LOB_DATA_FILE_NOT_FOUND_ERROR) + if (intParam1 == LOB_DATA_READ_ERROR) { - diagsArea << DgSqlCode(-EXE_TABLE_NOT_FOUND) - << DgString0(si->tableName()); + if ((failedLocBufLen > 0) && (strlen(failedLocBuf) > 0)) + { + char errBuf[strlen(si->tableName()) + 100 + failedLocBufLen]; + str_sprintf(errBuf, "%s (fileLoc: %s)", + si->tableName(), failedLocBuf); + diagsArea << DgSqlCode(-EXE_TABLE_NOT_FOUND) + << DgString0(errBuf); + } + else + { + diagsArea << DgSqlCode(-EXE_TABLE_NOT_FOUND) + << DgString0(si->tableName()); + } + simCheckFailed = TRUE; } return ERROR; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/executor/ExExeUtilLoad.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp index 57345bf..a945556 100644 --- a/core/sql/executor/ExExeUtilLoad.cpp +++ b/core/sql/executor/ExExeUtilLoad.cpp @@ -2758,7 +2758,8 @@ void ExExeUtilLobExtractTcb::freeResources() getGlobals()->castToExExeStmtGlobals()->castToExMasterStmtGlobals()-> getStatement()->getContext(); //close any open cursors. - retcode = ExpLOBInterfaceSelectCursor + if (lobHandle_ and lobName_) + retcode = ExpLOBInterfaceSelectCursor (lobGlobs, lobName_, lobLoc_, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/executor/ExHdfsScan.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp index 3b9baa5..00a673a 100644 --- a/core/sql/executor/ExHdfsScan.cpp +++ b/core/sql/executor/ExHdfsScan.cpp @@ -619,10 +619,16 @@ ExWorkProcRetcode ExHdfsScanTcb::work() { ComDiagsArea * diagsArea = NULL; if (hdfsErrorDetail == ENOENT) - ExRaiseSqlError(getHeap(), &diagsArea, + { + char errBuf[strlen(hdfsScanTdb().tableName()) + + strlen(hdfsFileName_) + 100]; + str_sprintf(errBuf, "%s (fileLoc: %s)", + hdfsScanTdb().tableName(), hdfsFileName_); + ExRaiseSqlError(getHeap(), &diagsArea, (ExeErrorCode)(EXE_TABLE_NOT_FOUND), NULL, NULL, NULL, NULL, - hdfsScanTdb().tableName()); + errBuf); + } else ExRaiseSqlError(getHeap(), &diagsArea, (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR)); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/exp/ExpLOBaccess.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp index 455fec1..6eced07 100644 --- a/core/sql/exp/ExpLOBaccess.cpp +++ b/core/sql/exp/ExpLOBaccess.cpp @@ -451,8 +451,25 @@ Ex_Lob_Error ExLob::dataModCheck( // find mod time of root dir hdfsFileInfo *fileInfos = hdfsGetPathInfo(fs_, dirPath); if (fileInfos == NULL) - { - return LOB_DATA_FILE_NOT_FOUND_ERROR; + { + Lng32 failedFileLen = strlen(dirPath); + Lng32 copyLen = (failedFileLen > (*failedLocBufLen-1) + ? (*failedLocBufLen-1) : failedFileLen); + Int32 hdfserror = errno; + char hdfsErrStr[20]; + sprintf(hdfsErrStr,"(errno %d)",errno); + str_cpy_and_null(failedLocBuf, dirPath, copyLen, + '\0', ' ', TRUE); + str_cat_c(failedLocBuf,hdfsErrStr); + *failedLocBufLen = copyLen; + if (errno) + { + // Allow for hdfs error. AQR will find the new hive mapped files + // if the hive table has been remapped to new data files + return LOB_DATA_FILE_NOT_FOUND_ERROR; + } + else + return LOB_DATA_READ_ERROR; } Int64 currModTS = fileInfos[0].mLastMod; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/optimizer/ItemFunc.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/ItemFunc.h b/core/sql/optimizer/ItemFunc.h index e4bdf07..c60447d 100644 --- a/core/sql/optimizer/ItemFunc.h +++ b/core/sql/optimizer/ItemFunc.h @@ -2921,7 +2921,7 @@ public: short lobNum_; LobsStorage lobStorageType_; NAString lobStorageLocation_; - Int32 lobMaxSize_; // In MB units + Int64 lobMaxSize_; // In byte units Int32 lobMaxChunkMemSize_; //In MB Units Int32 lobGCLimit_ ;//In MB Units Int32 hdfsPort_; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/regress/hive/DIFF003.KNOWN ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/DIFF003.KNOWN b/core/sql/regress/hive/DIFF003.KNOWN index 3480a0d..f8803ea 100644 --- a/core/sql/regress/hive/DIFF003.KNOWN +++ b/core/sql/regress/hive/DIFF003.KNOWN @@ -1,10 +1,50 @@ -192,195d191 -< *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. +543,546d542 +< *** WARNING[6008] Statistics for column (SS_SOLD_DATE_SK) from table HIVE.HIVE.STORE_SALES were not available. As a result, the access path chosen might not be the best possible. < -< *** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: removed +< *** WARNING[6008] Statistics for column (SS_STORE_SK) from table HIVE.HIVE.STORE_SALES were not available. As a result, the access path chosen might not be the best possible. < -216a213,216 -> *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -> -> *** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: removed -> +555,560c551,557 +< 6 . 7 root 2.92E+006 +< 5 . 6 esp_exchange 1:2(hash2) 2.92E+006 +< 3 . 5 hive_insert INS_STORE_SALES_SUMM 2.92E+006 +< 2 . 3 hash_groupby 2.92E+006 +< 1 . 2 esp_exchange 2(hash2):2(hash2) 2.92E+006 +< . . 1 hive_scan STORE_SALES 2.92E+006 +--- +> 7 . 8 root 1.09E+004 +> 6 . 7 esp_exchange 1:2(hash2) 1.09E+004 +> 4 . 6 hive_insert INS_STORE_SALES_SUMM 1.09E+004 +> 3 . 4 hash_partial_groupby 1.09E+004 +> 2 . 3 esp_exchange 2(hash2):2(hash2) 1.09E+004 +> 1 . 2 hash_partial_groupby 1.09E+004 +> . . 1 hive_scan STORE_SALES 2.75E+006 +597,601c594 +< 3,10c3,4 +< < *** WARNING[6008] Statistics for column (SS_SOLD_DATE_SK) from table HIVE.HIVE.STORE_SALES were not available. As a result, the access path chosen might not be the best possible. +< < +< < *** WARNING[6008] Statistics for column (SS_STORE_SK) from table HIVE.HIVE.STORE_SALES were not available. As a result, the access path chosen might not be the best possible. +< < +--- +> 3,6c3,4 +615,618d607 +< *** WARNING[6008] Statistics for column (SS_SOLD_DATE_SK) from table HIVE.HIVE.STORE_SALES were not available. As a result, the access path chosen might not be the best possible. +< +< *** WARNING[6008] Statistics for column (SS_STORE_SK) from table HIVE.HIVE.STORE_SALES were not available. As a result, the access path chosen might not be the best possible. +< +627,633c616,623 +< 8 . 9 root 2.92E+006 +< 1 7 8 blocked_union 2.92E+006 +< 6 . 7 esp_exchange 1:2(hash2) 2.92E+006 +< 4 . 6 hive_insert INS_STORE_SALES_SUMM 2.92E+006 +< 3 . 4 hash_groupby 2.92E+006 +< 2 . 3 esp_exchange 2(hash2):2(hash2) 2.92E+006 +< . . 2 hive_scan STORE_SALES 2.92E+006 +--- +> 9 . 10 root 1.09E+004 +> 1 8 9 blocked_union 1.09E+004 +> 7 . 8 esp_exchange 1:2(hash2) 1.09E+004 +> 5 . 7 hive_insert INS_STORE_SALES_SUMM 1.09E+004 +> 4 . 5 hash_partial_groupby 1.09E+004 +> 3 . 4 esp_exchange 2(hash2):2(hash2) 1.09E+004 +> 2 . 3 hash_partial_groupby 1.09E+004 +> . . 2 hive_scan STORE_SALES 2.75E+006 http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/regress/hive/FILTER005 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/FILTER005 b/core/sql/regress/hive/FILTER005 index b128157..08be9a6 100755 --- a/core/sql/regress/hive/FILTER005 +++ b/core/sql/regress/hive/FILTER005 @@ -31,4 +31,6 @@ sed " s/Logging Location: \/user\/trafodion\/bulkload\/logs\/ERR.*/Logging Location:@loggingLocation@/ s/Logging Location: \/user\/trafodion\/bulkload\/logs\/TEST005\/ERR.*/Logging Location:@TEST005LoggingLocation@/ s/Time:.*/Time:@time@/ +s/^\*\*\* WARNING\[8436\].*/Warning 8436 or 8577 removed/g +s/^\*\*\* WARNING\[8577\].*/Warning 8436 or 8577 removed/g " $fil
