Repository: incubator-trafodion Updated Branches: refs/heads/master ad6e7528a -> 60169923e
[TRAFODION-2351] Bulk load with log error rows enhancements Load with log error rows won't work if there are multiple ranges involved in the hdfs scan. The hdfs log file is closed when the first range of hdfs scan is done. The error rows in the subsequent ranges will not be logged. Instead an SQL warning is generated for each process. When these warnings are shipped from ESPs to master, ESP dumps core. Added tests to test load with log error rows and continue on error Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/3fb90984 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/3fb90984 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/3fb90984 Branch: refs/heads/master Commit: 3fb90984912be0242e10264306f2447aa94e91fd Parents: 660c80e Author: selvaganesang <[email protected]> Authored: Thu Jan 26 00:55:21 2017 +0000 Committer: selvaganesang <[email protected]> Committed: Thu Jan 26 00:55:21 2017 +0000 ---------------------------------------------------------------------- core/sql/executor/ExExeUtilLoad.cpp | 3 +- core/sql/executor/ExHdfsScan.cpp | 3 +- core/sql/regress/hive/EXPECTED005 | 149 ++++++++++++++++++++++++++++--- core/sql/regress/hive/TEST005 | 13 +++ 4 files changed, 154 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3fb90984/core/sql/executor/ExExeUtilLoad.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp index c19762f..6b8e6ac 100644 --- a/core/sql/executor/ExExeUtilLoad.cpp +++ b/core/sql/executor/ExExeUtilLoad.cpp @@ -1442,7 +1442,8 @@ short ExExeUtilHBaseBulkLoadTcb::work() ComCondition *cond; Lng32 entryNumber; while ((cond = diagsArea->findCondition(EXE_ERROR_ROWS_FOUND, &entryNumber)) != NULL) { - errorRowCount = cond->getOptionalInteger(0); + if (errorRowCount < cond->getOptionalInteger(0)) + errorRowCount = cond->getOptionalInteger(0); diagsArea->deleteWarning(entryNumber); } diagsArea->setRowCount(0); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3fb90984/core/sql/executor/ExHdfsScan.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp index 75d22db..a51f943 100644 --- a/core/sql/executor/ExHdfsScan.cpp +++ b/core/sql/executor/ExHdfsScan.cpp @@ -1463,7 +1463,6 @@ ExWorkProcRetcode ExHdfsScanTcb::work() getLobErrStr(intParam1)); pentry_down->setDiagsArea(diagsArea); } - retcode = ehi_->hdfsClose(); } if (step_ == CLOSE_FILE) { @@ -1489,6 +1488,8 @@ ExWorkProcRetcode ExHdfsScanTcb::work() { if (qparent_.up->isFull()) return WORK_OK; + if (ehi_ != NULL) + retcode = ehi_->hdfsClose(); ex_queue_entry *up_entry = qparent_.up->getTailEntry(); up_entry->copyAtp(pentry_down); up_entry->upState.parentIndex = http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3fb90984/core/sql/regress/hive/EXPECTED005 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED005 b/core/sql/regress/hive/EXPECTED005 index 665a151..ba76df3 100644 --- a/core/sql/regress/hive/EXPECTED005 +++ b/core/sql/regress/hive/EXPECTED005 @@ -72,7 +72,7 @@ *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: genModTS = 1472657273, failedModTS = 1472657341 +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1485388879, failedModTS = 1485388945, failedLoc = hdfs://localhost:27000/user/hive/exttables/customer_ddl C_PREFERRED_CUST_FLAG (EXPR) ------------------------- -------------------- @@ -108,7 +108,7 @@ Y 9525 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: genModTS = 1472657273, failedModTS = 1472657341 +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1485388879, failedModTS = 1485388945, failedLoc = hdfs://localhost:27000/user/hive/exttables/customer_ddl C_PREFERRED_CUST_FLAG (EXPR) ------------------------- -------------------- @@ -177,7 +177,7 @@ Y 9525 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: genModTS = 1472657345, failedModTS = 1472657357 +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1485388949, failedModTS = 1485388954, failedLoc = hdfs://localhost:27000/user/hive/warehouse/newtable A ------------------------- @@ -227,7 +227,7 @@ xyz *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: genModTS = 1472657341, failedModTS = 1472657378 +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1485388945, failedModTS = 1485388975, failedLoc = hdfs://localhost:27000/user/hive/exttables/customer_ddl C_PREFERRED_CUST_FLAG (EXPR) ------------------------- -------------------- @@ -242,7 +242,7 @@ Y 18984 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: genModTS = 1472657341, failedModTS = 1472657378 +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1485388945, failedModTS = 1485388975, failedLoc = hdfs://localhost:27000/user/hive/exttables/customer_ddl C_PREFERRED_CUST_FLAG (EXPR) ------------------------- -------------------- @@ -298,7 +298,7 @@ Y 18984 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: genModTS = 1472657375, failedModTS = 1472657383 +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1485388971, failedModTS = 1485388977, failedLoc = hdfs://localhost:27000/user/hive/warehouse/newtable A B ----------- ------------------------- @@ -607,6 +607,131 @@ C1 C2 C3 C4 C *** ERROR[8413] The string argument contains characters that cannot be converted. Source data(in hex): 61 --- 0 row(s) loaded. +>>select count(*) from trafodion.seabase.traf_tbl_bad; + +(EXPR) +-------------------- + + 0 + +--- 1 row(s) selected. +>>load with continue on error into trafodion.seabase.traf_tbl_bad select * from tbl_bad; +Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD +Task: CLEANUP Status: Started Time: 2017-01-26 00:04:00.936 +Task: CLEANUP Status: Ended Time: 2017-01-26 00:04:00.954 +Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.018 +Task: LOADING DATA Status: Started Time: 2017-01-26 00:04:00.955 + Rows Processed: 8 + Error Rows: 5 +Task: LOADING DATA Status: Ended Time: 2017-01-26 00:04:01.231 +Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.276 +Task: COMPLETION Status: Started Time: 2017-01-26 00:04:01.231 + Rows Loaded: 3 +Task: COMPLETION Status: Ended Time: 2017-01-26 00:04:01.663 +Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.432 + +--- 3 row(s) loaded. +>>select count(*) from trafodion.seabase.traf_tbl_bad; + +(EXPR) +-------------------- + + 3 + +--- 1 row(s) selected. +>>delete from trafodion.seabase.traf_tbl_bad ; + +--- 3 row(s) deleted. +>>load with log error rows into trafodion.seabase.traf_tbl_bad select * from tbl_bad; +Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD +Task: CLEANUP Status: Started Time: 2017-01-26 00:04:02.707 +Task: CLEANUP Status: Ended Time: 2017-01-26 00:04:02.714 +Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.007 + Logging Location: /bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20170126_000402 +Task: LOADING DATA Status: Started Time: 2017-01-26 00:04:02.714 + Rows Processed: 8 + Error Rows: 5 +Task: LOADING DATA Status: Ended Time: 2017-01-26 00:04:03.117 +Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.404 +Task: COMPLETION Status: Started Time: 2017-01-26 00:04:03.117 + Rows Loaded: 3 +Task: COMPLETION Status: Ended Time: 2017-01-26 00:04:03.702 +Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.585 + +--- 3 row(s) loaded. +>>select count(*) from trafodion.seabase.traf_tbl_bad; + +(EXPR) +-------------------- + + 3 + +--- 1 row(s) selected. +>>load with log error rows to '/bulkload/logs/TEST005' into trafodion.seabase.traf_tbl_bad select * from tbl_bad; +Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD +Task: CLEANUP Status: Started Time: 2017-01-26 00:04:04.711 +Task: CLEANUP Status: Ended Time: 2017-01-26 00:04:04.724 +Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.013 + Logging Location: /bulkload/logs/TEST005/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20170126_000404 +Task: LOADING DATA Status: Started Time: 2017-01-26 00:04:04.724 + Rows Processed: 8 + Error Rows: 5 +Task: LOADING DATA Status: Ended Time: 2017-01-26 00:04:05.154 +Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.430 +Task: COMPLETION Status: Started Time: 2017-01-26 00:04:05.154 + Rows Loaded: 3 +Task: COMPLETION Status: Ended Time: 2017-01-26 00:04:05.752 +Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.599 + +--- 3 row(s) loaded. +>>select count(*) from trafodion.seabase.traf_tbl_bad; + +(EXPR) +-------------------- + + 6 + +--- 1 row(s) selected. +>>delete from trafodion.seabase.traf_tbl_bad ; + +--- 6 row(s) deleted. +>>load with stop after 3 error rows into trafodion.seabase.traf_tbl_bad select * from tbl_bad; +Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD +Task: CLEANUP Status: Started Time: 2017-01-26 00:04:06.808 +Task: CLEANUP Status: Ended Time: 2017-01-26 00:04:06.815 +Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.007 +Task: LOADING DATA Status: Started Time: 2017-01-26 00:04:06.815 + +*** ERROR[8113] The maximum number of error rows is exceeded. + +--- 0 row(s) loaded. +>>select count(*) from trafodion.seabase.traf_tbl_bad; + +(EXPR) +-------------------- + + 0 + +--- 1 row(s) selected. +>>load with log error rows, stop after 3 error rows into trafodion.seabase.traf_tbl_bad select * from tbl_bad; +Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD +Task: CLEANUP Status: Started Time: 2017-01-26 00:04:08.538 +Task: CLEANUP Status: Ended Time: 2017-01-26 00:04:08.668 +Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.013 + Logging Location: /bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20170126_000408 +Task: LOADING DATA Status: Started Time: 2017-01-26 00:04:08.669 + +*** ERROR[8113] The maximum number of error rows is exceeded. + +--- 0 row(s) loaded. +>>select count(*) from trafodion.seabase.traf_tbl_bad; + +(EXPR) +-------------------- + + 0 + +--- 1 row(s) selected. >>insert into trafodion.seabase.traf_tbl_bad select * from tbl_bad; *** ERROR[8413] The string argument contains characters that cannot be converted. Source data(in hex): 61 @@ -681,7 +806,7 @@ C1 C2 C3 C4 C *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: genModTS = 1472657488, failedModTS = 1472657504 +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1485389081, failedModTS = 1485389096, failedLoc = hdfs://localhost:27000/user/hive/warehouse/thive A ----------- @@ -715,7 +840,7 @@ A *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: genModTS = 1472657505, failedModTS = 1472657542 +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1485389099, failedModTS = 1485389135, failedLoc = hdfs://localhost:27000/user/hive/warehouse/thive A ----------- @@ -742,7 +867,7 @@ A B *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: genModTS = 1472657542, failedModTS = 1472657545 +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1485389135, failedModTS = 1485389138, failedLoc = hdfs://localhost:27000/user/hive/warehouse/thive --- 0 row(s) selected. >>insert into hive.hive.thive values (10, 20); @@ -766,7 +891,7 @@ A B *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: genModTS = 1472657548, failedModTS = 1472657549 +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1485389140, failedModTS = 1485389142, failedLoc = hdfs://localhost:27000/user/hive/warehouse/thive --- 0 row(s) selected. >> @@ -856,7 +981,7 @@ t005part.a t005part.b t005part.c >>invoke hive.hive.thive_insert_smallint; -- Definition of hive table THIVE_INSERT_SMALLINT --- Definition current Wed Aug 31 15:35:21 2016 +-- Definition current Thu Jan 26 00:08:37 2017 ( A SMALLINT @@ -961,7 +1086,7 @@ A >>invoke hive.hive.thive_insert_varchar; -- Definition of hive table THIVE_INSERT_VARCHAR --- Definition current Wed Aug 31 15:35:26 2016 +-- Definition current Thu Jan 26 00:08:42 2017 ( A VARCHAR(1 CHAR) CHARACTER SET UTF8 COLLATE http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3fb90984/core/sql/regress/hive/TEST005 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST005 b/core/sql/regress/hive/TEST005 index 8a4c8b1..3d6ef81 100644 --- a/core/sql/regress/hive/TEST005 +++ b/core/sql/regress/hive/TEST005 @@ -260,6 +260,19 @@ c7 float(54), c8 smallint ); load with no output into trafodion.seabase.traf_tbl_bad select * from tbl_bad; +select count(*) from trafodion.seabase.traf_tbl_bad; +load with continue on error into trafodion.seabase.traf_tbl_bad select * from tbl_bad; +select count(*) from trafodion.seabase.traf_tbl_bad; +delete from trafodion.seabase.traf_tbl_bad ; +load with log error rows into trafodion.seabase.traf_tbl_bad select * from tbl_bad; +select count(*) from trafodion.seabase.traf_tbl_bad; +load with log error rows to '/bulkload/logs/TEST005' into trafodion.seabase.traf_tbl_bad select * from tbl_bad; +select count(*) from trafodion.seabase.traf_tbl_bad; +delete from trafodion.seabase.traf_tbl_bad ; +load with stop after 3 error rows into trafodion.seabase.traf_tbl_bad select * from tbl_bad; +select count(*) from trafodion.seabase.traf_tbl_bad; +load with log error rows, stop after 3 error rows into trafodion.seabase.traf_tbl_bad select * from tbl_bad; +select count(*) from trafodion.seabase.traf_tbl_bad; insert into trafodion.seabase.traf_tbl_bad select * from tbl_bad; upsert into trafodion.seabase.traf_tbl_bad select * from tbl_bad; cqd HIVE_SCAN_SPECIAL_MODE '2';
