Repository: incubator-trafodion Updated Branches: refs/heads/master a2daeeea7 -> 3a7b00563
JIRA TRAFODION-2022 Add support for purgedata of hive tables In addition to purgedata: -- showplan correctly shows aqr setting -- data modification check is skipped during insert if overwrite option was used Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/c8a6187f Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/c8a6187f Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/c8a6187f Branch: refs/heads/master Commit: c8a6187f08d8bcb0074630b2e30fea723c53733c Parents: a2daeee Author: Anoop Sharma <[email protected]> Authored: Thu Jun 2 14:52:21 2016 +0000 Committer: Anoop Sharma <[email protected]> Committed: Thu Jun 2 14:52:21 2016 +0000 ---------------------------------------------------------------------- core/sql/comexe/ComTdbFastTransport.cpp | 3 + core/sql/executor/ExExeUtilMisc.cpp | 34 +---- core/sql/executor/ExFastTransport.cpp | 6 +- core/sql/executor/ExHdfsScan.cpp | 4 +- core/sql/generator/Generator.cpp | 3 +- core/sql/optimizer/RelExeUtil.cpp | 39 +++++- core/sql/regress/hive/EXPECTED003 | 28 ++-- core/sql/regress/hive/EXPECTED005 | 31 +++++ core/sql/regress/hive/EXPECTED018 | 184 +++++++++++++-------------- core/sql/regress/hive/TEST005 | 10 ++ core/sql/regress/tools/runallsb | 2 +- 11 files changed, 188 insertions(+), 156 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c8a6187f/core/sql/comexe/ComTdbFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/comexe/ComTdbFastTransport.cpp b/core/sql/comexe/ComTdbFastTransport.cpp index f492e8f..bca4d32 100644 --- a/core/sql/comexe/ComTdbFastTransport.cpp +++ b/core/sql/comexe/ComTdbFastTransport.cpp @@ -220,6 +220,9 @@ void ComTdbFastExtract::displayContents(Space *space, ULng32 flag) str_sprintf(buf,"numIOBuffers = %d", numIOBuffers_); space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(UInt16)); + str_sprintf(buf, "modTSforDir_ = %Ld", modTSforDir_); + space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(short)); + } // if (flag & 0x00000008) displayExpression(space,flag); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c8a6187f/core/sql/executor/ExExeUtilMisc.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtilMisc.cpp b/core/sql/executor/ExExeUtilMisc.cpp index 267d22e..9c86f72 100644 --- a/core/sql/executor/ExExeUtilMisc.cpp +++ b/core/sql/executor/ExExeUtilMisc.cpp @@ -2258,7 +2258,7 @@ Int32 ExExeUtilHiveTruncateTcb::fixup() return 0; } ////////////////////////////////////////////////////// -// work() for ExExePurgedataUtilTcb +// work() for ExExeUtilHiveTruncateTsb ////////////////////////////////////////////////////// short ExExeUtilHiveTruncateTcb::work() { @@ -2376,28 +2376,15 @@ short ExExeUtilHiveTruncateTcb::work() if (qparent_.up->isFull()) return WORK_OK; - // Return EOF. + // Return Error ex_queue_entry * up_entry = qparent_.up->getTailEntry(); + up_entry->copyAtp(pentry_down); up_entry->upState.parentIndex = pentry_down->downState.parentIndex; up_entry->upState.setMatchNo(0); up_entry->upState.status = ex_queue::Q_SQLERROR; - ComDiagsArea *diagsArea = up_entry->getDiagsArea(); - - if (diagsArea == NULL) - diagsArea = ComDiagsArea::allocate(this->getGlobals()->getDefaultHeap()); - else - diagsArea->incrRefCount(); // setDiagsArea call below will decr ref count - - if (getDiagsArea()) - diagsArea->mergeAfter(*getDiagsArea()); - - up_entry->setDiagsArea(diagsArea); - - getDiagsArea()->clear(); - // insert into parent qparent_.up->insert(); @@ -2418,21 +2405,6 @@ short ExExeUtilHiveTruncateTcb::work() up_entry->upState.setMatchNo(0); up_entry->upState.status = ex_queue::Q_NO_DATA; - if (getDiagsArea()->getNumber(DgSqlCode::WARNING_) > 0) // must be a warning - { - ComDiagsArea *diagsArea = up_entry->getDiagsArea(); - - if (diagsArea == NULL) - diagsArea = ComDiagsArea::allocate(this->getGlobals()->getDefaultHeap()); - else - diagsArea->incrRefCount(); // setDiagsArea call below will decr ref count - - if (getDiagsArea()) - diagsArea->mergeAfter(*getDiagsArea()); - - up_entry->setDiagsArea(diagsArea); - } - // insert into parent qparent_.up->insert(); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c8a6187f/core/sql/executor/ExFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExFastTransport.cpp b/core/sql/executor/ExFastTransport.cpp index bb8d4dc..2ebba4b 100644 --- a/core/sql/executor/ExFastTransport.cpp +++ b/core/sql/executor/ExFastTransport.cpp @@ -698,8 +698,12 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work() case EXTRACT_CHECK_MOD_TS: { + // if no tgt file or input timestamp is -1, skip data mod check. + // Also, if this insert is being done with overwrite, then data mod + // check has already been done during directory cleanup. Skip it here. if ((! myTdb().getTargetFile()) || - (myTdb().getModTSforDir() == -1)) + (myTdb().getModTSforDir() == -1) || + (myTdb().getOverwriteHiveTable())) { pstate.step_ = EXTRACT_INITIALIZE; break; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c8a6187f/core/sql/executor/ExHdfsScan.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp index 7617e70..badccf4 100644 --- a/core/sql/executor/ExHdfsScan.cpp +++ b/core/sql/executor/ExHdfsScan.cpp @@ -425,12 +425,12 @@ ExWorkProcRetcode ExHdfsScanTcb::work() case CHECK_FOR_DATA_MOD_AND_DONE: { char * dirPath = hdfsScanTdb().hdfsRootDir_; - if (! dirPath) + Int64 modTS = hdfsScanTdb().modTSforDir_; + if ((dirPath == NULL) || (modTS == -1)) dataModCheckDone_ = TRUE; if (NOT dataModCheckDone_) { - Int64 modTS = hdfsScanTdb().modTSforDir_; Lng32 numOfPartLevels = hdfsScanTdb().numOfPartCols_; if (hdfsScanTdb().hdfsDirsToCheck()) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c8a6187f/core/sql/generator/Generator.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/Generator.cpp b/core/sql/generator/Generator.cpp index f595192..b0d5f3b 100644 --- a/core/sql/generator/Generator.cpp +++ b/core/sql/generator/Generator.cpp @@ -517,7 +517,8 @@ RelExpr * Generator::preGenCode(RelExpr * expr_node) // if internal query from executor for explain, enable aqr. const NAString * val = ActiveControlDB()->getControlSessionValue("EXPLAIN"); - if ((val) && (*val == "ON")) + if (((val) && (*val == "ON")) || + (exp_generator->getShowplan())) { aqr = TRUE; } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c8a6187f/core/sql/optimizer/RelExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExeUtil.cpp b/core/sql/optimizer/RelExeUtil.cpp index f596474..0b439ae 100644 --- a/core/sql/optimizer/RelExeUtil.cpp +++ b/core/sql/optimizer/RelExeUtil.cpp @@ -4981,9 +4981,43 @@ RelExpr * ExeUtilFastDelete::bindNode(BindWA *bindWA) { // do not do override schema for this bindWA->setToOverrideSchema(FALSE); - + naTable = bindWA->getNATable(getTableName()); - if (getTableName().isSeabase()) + if (getTableName().isHive()) + { + if (! naTable) + { + *CmpCommon::diags() << DgSqlCode(-4222) << DgString0("PURGEDATA"); + bindWA->setErrStatus(); + return NULL; + } + + const HHDFSTableStats* hTabStats = + naTable->getClusteringIndex()->getHHDFSTableStats(); + + isHiveTable_ = TRUE; + + const char * hiveTablePath = (*hTabStats)[0]->getDirName(); + NAString hostName; + Int32 hdfsPort; + NAString tableDir; + + NABoolean result = ((HHDFSTableStats* )hTabStats)->splitLocation + (hiveTablePath, hostName, hdfsPort, tableDir) ; + if (!result) + { + *CmpCommon::diags() << DgSqlCode(-4224) + << DgString0(hiveTablePath); + bindWA->setErrStatus(); + return this; + } + + hiveTableLocation_ = tableDir; + hiveHostName_ = hostName; + hiveHdfsPort_ = hdfsPort; + hiveModTS_ = -1; + } + else if (getTableName().isSeabase()) { if (bindWA->errStatus()) return this; @@ -5001,7 +5035,6 @@ RelExpr * ExeUtilFastDelete::bindNode(BindWA *bindWA) naTable = NULL; CmpCommon::diags()->clear(); bindWA->resetErrStatus(); - } } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c8a6187f/core/sql/regress/hive/EXPECTED003 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED003 b/core/sql/regress/hive/EXPECTED003 index 0d0c3d0..21fef14 100644 --- a/core/sql/regress/hive/EXPECTED003 +++ b/core/sql/regress/hive/EXPECTED003 @@ -177,10 +177,6 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S >> >>insert OVERWRITE TABLE hive.ins_customer_address select * from >>hive.customer_address; -*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. - -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. - --- 50000 row(s) inserted. >> >>select count(*) from hive.customer_address; @@ -193,6 +189,10 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S --- 1 row(s) selected. >>select count(*) from hive.ins_customer_address; +*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. + +*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. + (EXPR) -------------------- @@ -202,10 +202,6 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S >>--execute again >>insert OVERWRITE TABLE hive.ins_customer_address select * from >>hive.customer_address; -*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. - -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. - --- 50000 row(s) inserted. >> >>select count(*) from hive.customer_address; @@ -637,10 +633,6 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD --- SQL operation complete. >>execute s; -*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. - -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. - --- 12768 row(s) inserted. >>control query shape cut; @@ -650,15 +642,13 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD < >>select ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales group by ss_sold_date_sk ,ss_store_sk order by ss_sold_date_sk,ss_store_sk; --- > >>select * from ins_store_sales_summary order by > >>ss_sold_date_sk,ss_store_sk; -3,10c3,4 -< *** WARNING[6008] Statistics for column (SS_SOLD_DATE_SK) from table HIVE.HIVE.STORE_SALES were not available. As a result, the access path chosen might not be the best possible. -< -< *** WARNING[6008] Statistics for column (SS_STORE_SK) from table HIVE.HIVE.STORE_SALES were not available. As a result, the access path chosen might not be the best possible. -< -< *** WARNING[6007] Multi-column statistics for columns (SS_SOLD_DATE_SK, SS_STORE_SK) from table HIVE.HIVE.STORE_SALES were not available. The columns were being used by GroupBy operator. As a result, the access path chosen might not be the best possible. -< +3,4c3,8 < SS_SOLD_DATE_SK SS_STORE_SK (EXPR) < --------------- ----------- -------------------- --- +> *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. +> +> *** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. +> > SS_SOLD_DATE_SK SS_STORE_SK SS_QUANTITY > --------------- ----------- ----------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c8a6187f/core/sql/regress/hive/EXPECTED005 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED005 b/core/sql/regress/hive/EXPECTED005 index eaa414e..6e65f1a 100644 --- a/core/sql/regress/hive/EXPECTED005 +++ b/core/sql/regress/hive/EXPECTED005 @@ -710,4 +710,35 @@ A B --- 1 row(s) selected. >> +>>-- purgedata of hive data +>>cqd query_cache '0'; + +--- SQL operation complete. +>>purgedata hive.hive.thive; + +--- SQL operation complete. +>>select * from hive.hive.thive; + +--- 0 row(s) selected. +>>insert into hive.hive.thive values (10, 20); + +--- 1 row(s) inserted. +>>select * from hive.hive.thive; + +A B +----------- ----------- + + 10 20 + +--- 1 row(s) selected. +>>purgedata hive.hive.thive; + +--- SQL operation complete. +>>purgedata hive.hive.thive; + +--- SQL operation complete. +>>select * from hive.hive.thive; + +--- 0 row(s) selected. +>> >>log; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c8a6187f/core/sql/regress/hive/EXPECTED018 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED018 b/core/sql/regress/hive/EXPECTED018 index 2c34c31..cff41c7 100644 --- a/core/sql/regress/hive/EXPECTED018 +++ b/core/sql/regress/hive/EXPECTED018 @@ -150,9 +150,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRE Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_ADDRESS Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS Rows Processed: 50000 -Task: PREPARATION Status: Ended ET: 00:00:06.864 +Task: PREPARATION Status: Ended ET: 00:00:08.543 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS -Task: COMPLETION Status: Ended ET: 00:00:00.262 +Task: COMPLETION Status: Ended ET: 00:00:00.256 --- 50000 row(s) loaded. >>-- @@ -181,9 +181,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS Rows Processed: 20000 -Task: PREPARATION Status: Ended ET: 00:00:10.758 +Task: PREPARATION Status: Ended ET: 00:00:11.022 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS -Task: COMPLETION Status: Ended ET: 00:00:00.234 +Task: COMPLETION Status: Ended ET: 00:00:00.196 --- 20000 row(s) loaded. >>-- @@ -213,9 +213,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT Rows Processed: 20000 -Task: PREPARATION Status: Ended ET: 00:00:09.668 +Task: PREPARATION Status: Ended ET: 00:00:07.143 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT -Task: COMPLETION Status: Ended ET: 00:00:00.208 +Task: COMPLETION Status: Ended ET: 00:00:00.177 --- 20000 row(s) loaded. >>-- >> @@ -235,9 +235,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_SALT Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT Rows Processed: 100000 -Task: PREPARATION Status: Ended ET: 00:00:07.549 +Task: PREPARATION Status: Ended ET: 00:00:07.625 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT -Task: COMPLETION Status: Ended ET: 00:00:00.167 +Task: COMPLETION Status: Ended ET: 00:00:00.199 --- 100000 row(s) loaded. >>-- @@ -266,9 +266,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.STORE_SALES_SA Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.STORE_SALES_SALT Task: PREPARATION Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT Rows Processed: 160756 -Task: PREPARATION Status: Ended ET: 00:00:11.382 +Task: PREPARATION Status: Ended ET: 00:00:10.636 Task: COMPLETION Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT -Task: COMPLETION Status: Ended ET: 00:00:00.180 +Task: COMPLETION Status: Ended ET: 00:00:00.208 --- 160756 row(s) loaded. >>-- @@ -288,10 +288,6 @@ Task: COMPLETION Status: Ended ET: 00:00:00.180 >>-- using insert >>insert overwrite table hive.hive.null_format_default select * from >>null_format_src; -*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. - -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. - --- 10 row(s) inserted. >>select * from hive.hive.null_format_default; @@ -313,10 +309,6 @@ a >> >>insert overwrite table hive.hive.null_format_empty select * from >>null_format_src; -*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. - -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. - --- 10 row(s) inserted. >>select * from hive.hive.null_format_empty; @@ -338,10 +330,6 @@ a ? >> >>insert overwrite table hive.hive.null_format_colon select * from >>null_format_src; -*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry. - -*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. - --- 10 row(s) inserted. >>select * from hive.hive.null_format_colon; @@ -368,10 +356,10 @@ a +> select * from null_format_src; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.002 Task: EXTRACT Status: Started Rows Processed: 10 -Task: EXTRACT Status: Ended ET: 00:00:00.232 +Task: EXTRACT Status: Ended ET: 00:00:00.238 --- 10 row(s) unloaded. >>select * from hive.hive.null_format_default; @@ -397,10 +385,10 @@ a +> select * from null_format_src; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.001 Task: EXTRACT Status: Started Rows Processed: 10 -Task: EXTRACT Status: Ended ET: 00:00:00.218 +Task: EXTRACT Status: Ended ET: 00:00:00.369 --- 10 row(s) unloaded. >>select * from hive.hive.null_format_empty; @@ -426,10 +414,10 @@ a ? +> select * from null_format_src; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.002 Task: EXTRACT Status: Started Rows Processed: 10 -Task: EXTRACT Status: Ended ET: 00:00:00.213 +Task: EXTRACT Status: Ended ET: 00:00:00.225 --- 10 row(s) unloaded. >>select * from hive.hive.null_format_colon; @@ -483,12 +471,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.007 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.002 Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:03.805 +Task: EXTRACT Status: Ended ET: 00:00:02.759 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.034 +Task: MERGE FILES Status: Ended ET: 00:00:00.027 --- 50000 row(s) unloaded. >>log; @@ -521,12 +509,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.002 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.865 +Task: EXTRACT Status: Ended ET: 00:00:01.075 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.023 +Task: MERGE FILES Status: Ended ET: 00:00:00.028 --- 20000 row(s) unloaded. >>log; @@ -545,12 +533,12 @@ cat /tmp/merged_customer_demogs | wc -l +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.766 +Task: EXTRACT Status: Ended ET: 00:00:01.282 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.023 +Task: MERGE FILES Status: Ended ET: 00:00:00.026 --- 20000 row(s) unloaded. >>log; @@ -582,12 +570,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.003 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.855 +Task: EXTRACT Status: Ended ET: 00:00:00.850 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.032 +Task: MERGE FILES Status: Ended ET: 00:00:00.035 --- 20000 row(s) unloaded. >> @@ -605,12 +593,12 @@ regrhadoop.ksh fs -du -s /bulkload/customer_demographics_salt/merged_customer_de +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.692 +Task: EXTRACT Status: Ended ET: 00:00:00.673 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.031 +Task: MERGE FILES Status: Ended ET: 00:00:00.035 --- 20000 row(s) unloaded. >> @@ -643,10 +631,10 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.484 +Task: EXTRACT Status: Ended ET: 00:00:00.671 --- 20000 row(s) unloaded. >> @@ -666,12 +654,12 @@ regrhadoop.ksh fs -ls /bulkload/customer_demographics_salt/file* | grep file | +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.010 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.011 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.580 +Task: EXTRACT Status: Ended ET: 00:00:00.683 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.038 +Task: MERGE FILES Status: Ended ET: 00:00:00.035 --- 20000 row(s) unloaded. >> @@ -804,10 +792,10 @@ CD_DEMO_SK CD_GENDER +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.003 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.487 +Task: EXTRACT Status: Ended ET: 00:00:00.596 Task: MERGE FILES Status: Started Task: MERGE FILES Status: Ended ET: 00:00:00.030 @@ -847,9 +835,9 @@ Task: EMPTY TARGET Status: Started Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.492 +Task: EXTRACT Status: Ended ET: 00:00:00.622 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.033 +Task: MERGE FILES Status: Ended ET: 00:00:00.037 --- 20000 row(s) unloaded. >> @@ -899,10 +887,10 @@ CD_DEMO_SK CD_GENDER +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.980 +Task: EXTRACT Status: Ended ET: 00:00:00.594 --- 20000 row(s) unloaded. >> @@ -953,10 +941,10 @@ CD_DEMO_SK CD_GENDER +>select * from trafodion.hbase.customer_address ; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.007 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.003 Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:02.111 +Task: EXTRACT Status: Ended ET: 00:00:01.887 --- 50000 row(s) unloaded. >> @@ -1006,10 +994,10 @@ CA_ADDRESS_SK CA_ADDRESS_ID +>select * from trafodion.hbase.customer_address ; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.003 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:02.075 +Task: EXTRACT Status: Ended ET: 00:00:02.001 --- 50000 row(s) unloaded. >> @@ -1072,10 +1060,10 @@ CA_ADDRESS_SK CA_ADDRESS_ID +>select * from trafodion.hbase.customer_salt; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.002 Task: EXTRACT Status: Started Rows Processed: 100000 -Task: EXTRACT Status: Ended ET: 00:00:06.402 +Task: EXTRACT Status: Ended ET: 00:00:06.708 --- 100000 row(s) unloaded. >>select count(*) from hive.hive.unload_customer; @@ -1125,10 +1113,10 @@ C_CUSTOMER_SK C_CUSTOMER_ID +>select * from trafodion.hbase.customer_demographics_salt; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.008 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.007 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.985 +Task: EXTRACT Status: Ended ET: 00:00:01.046 --- 20000 row(s) unloaded. >> @@ -1181,9 +1169,9 @@ Task: EMPTY TARGET Status: Started Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: EXTRACT Status: Started Rows Processed: 99 -Task: EXTRACT Status: Ended ET: 00:00:00.203 +Task: EXTRACT Status: Ended ET: 00:00:00.195 Task: MERGE FILES Status: Started -Task: MERGE FILES Status: Ended ET: 00:00:00.021 +Task: MERGE FILES Status: Ended ET: 00:00:00.019 --- 99 row(s) unloaded. >> @@ -1219,10 +1207,10 @@ regrhadoop.ksh fs -rm /user/hive/exttables/unload_customer_demographics/* +>select ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales_salt group by ss_sold_date_sk ,ss_store_sk; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.003 Task: EXTRACT Status: Started Rows Processed: 12349 -Task: EXTRACT Status: Ended ET: 00:00:06.351 +Task: EXTRACT Status: Ended ET: 00:00:06.194 --- 12349 row(s) unloaded. >> @@ -1341,10 +1329,10 @@ SS_SOLD_DATE_SK SS_STORE_SK SS_QUANTITY +>select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.002 Task: EXTRACT Status: Started Rows Processed: 100000 -Task: EXTRACT Status: Ended ET: 00:00:07.648 +Task: EXTRACT Status: Ended ET: 00:00:08.079 --- 100000 row(s) unloaded. >> @@ -1392,10 +1380,10 @@ C_CUSTOMER_SK C_CUSTOMER_ID +>select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.008 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.009 Task: EXTRACT Status: Started Rows Processed: 1998 -Task: EXTRACT Status: Ended ET: 00:00:00.490 +Task: EXTRACT Status: Ended ET: 00:00:00.545 --- 1998 row(s) unloaded. >> @@ -1506,7 +1494,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2 use_snapshot_scan ...... TRUE full_table_name ........ TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT snapshot_name .......... TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT_SNAP111 - snapshot_temp_location /bulkload/20160530055515/ + snapshot_temp_location /bulkload/20160602095555/ grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id >>--no snapshot >>explain reg; @@ -1586,7 +1574,7 @@ grep -i -e 'explain snp' -e snapshot -e full_table_name -e esp_exchange LOG018_S use_snapshot_scan ...... TRUE full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111 - snapshot_temp_location /bulkload/20160530055522/ + snapshot_temp_location /bulkload/20160602095603/ grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id >>--no snapshot >>explain reg; @@ -1668,11 +1656,11 @@ grep -i -e 'explain snp' -e snapshot -e full_table_name -e esp_exchange LOG018_S use_snapshot_scan ...... TRUE full_table_name ........ TRAFODION.HBASE.CUSTOMER_SALT snapshot_name .......... TRAFODION.HBASE.CUSTOMER_SALT_SNAP111 - snapshot_temp_location /bulkload/20160530055545/ + snapshot_temp_location /bulkload/20160602095624/ use_snapshot_scan ...... TRUE full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111 - snapshot_temp_location /bulkload/20160530055545/ + snapshot_temp_location /bulkload/20160602095624/ grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id >>--no snapshot >>explain reg; @@ -1785,13 +1773,13 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +><<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.003 Task: VERIFY SNAPSHO Status: Started Snapshots verified: 1 Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.294 Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:02.144 +Task: EXTRACT Status: Ended ET: 00:00:02.337 --- 50000 row(s) unloaded. >> @@ -1863,13 +1851,13 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.003 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.002 Task: VERIFY SNAPSHO Status: Started Snapshots verified: 1 -Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.311 +Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.272 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:02.081 +Task: EXTRACT Status: Ended ET: 00:00:02.527 --- 20000 row(s) unloaded. >> @@ -1917,16 +1905,16 @@ CD_DEMO_SK CD_GENDER +>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.010 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 Task: CREATE SNAPSHO Status: Started Snapshots created: 1 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.416 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:02.385 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:01.191 +Task: EXTRACT Status: Ended ET: 00:00:01.247 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 1 -Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.010 +Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.009 --- 20000 row(s) unloaded. >> @@ -1978,16 +1966,16 @@ CD_DEMO_SK CD_GENDER +>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.009 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.006 Task: CREATE SNAPSHO Status: Started Snapshots created: 1 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.602 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.561 Task: EXTRACT Status: Started Rows Processed: 20000 -Task: EXTRACT Status: Ended ET: 00:00:01.481 +Task: EXTRACT Status: Ended ET: 00:00:01.201 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 1 -Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.005 +Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.004 --- 20000 row(s) unloaded. >> @@ -2043,10 +2031,10 @@ Task: EMPTY TARGET Status: Started Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: CREATE SNAPSHO Status: Started Snapshots created: 1 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.627 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.958 Task: EXTRACT Status: Started Rows Processed: 1998 -Task: EXTRACT Status: Ended ET: 00:00:00.648 +Task: EXTRACT Status: Ended ET: 00:00:00.731 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 1 Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.004 @@ -2124,13 +2112,13 @@ CA_ADDRESS_SK CA_ADDRESS_ID +>select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.005 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: CREATE SNAPSHO Status: Started Snapshots created: 2 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:02.377 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:02.075 Task: EXTRACT Status: Started Rows Processed: 100000 -Task: EXTRACT Status: Ended ET: 00:00:08.169 +Task: EXTRACT Status: Ended ET: 00:00:08.534 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 2 Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.007 @@ -2201,16 +2189,16 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD +>select c_first_name,c_last_name from trafodion.hbase.customer_salt; Task: UNLOAD Status: Started Task: EMPTY TARGET Status: Started -Task: EMPTY TARGET Status: Ended ET: 00:00:00.007 +Task: EMPTY TARGET Status: Ended ET: 00:00:00.004 Task: CREATE SNAPSHO Status: Started Snapshots created: 1 -Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.420 +Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.415 Task: EXTRACT Status: Started Rows Processed: 100000 -Task: EXTRACT Status: Ended ET: 00:00:01.144 +Task: EXTRACT Status: Ended ET: 00:00:01.417 Task: DELETE SNAPSHO Status: Started Snapshots deleted: 1 -Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.004 +Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.003 --- 100000 row(s) unloaded. >> @@ -2283,7 +2271,7 @@ unload with delimiter 0 into '/bulkload/test' select * from CUSTOMER_ADDRESS; Task: UNLOAD Status: Started Task: EXTRACT Status: Started Rows Processed: 50000 -Task: EXTRACT Status: Ended ET: 00:00:01.974 +Task: EXTRACT Status: Ended ET: 00:00:01.957 --- 50000 row(s) unloaded. >>--unload 24 -- should give an error @@ -2348,7 +2336,7 @@ regrhadoop.ksh fs -rm /user/hive/exttables/unload_customer_demographics/* Task: UNLOAD Status: Started Task: EXTRACT Status: Started Rows Processed but NOT Written to Disk: 20000 -Task: EXTRACT Status: Ended ET: 00:00:00.690 +Task: EXTRACT Status: Ended ET: 00:00:00.806 --- 20000 row(s) unloaded. >>select count(*) from hive.hive.unload_customer_demographics; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c8a6187f/core/sql/regress/hive/TEST005 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/TEST005 b/core/sql/regress/hive/TEST005 index cd3fa56..2fe0b86 100644 --- a/core/sql/regress/hive/TEST005 +++ b/core/sql/regress/hive/TEST005 @@ -311,4 +311,14 @@ select a from hive.hive.thive; select * from hive.hive.thive; +-- purgedata of hive data +cqd query_cache '0'; +purgedata hive.hive.thive; +select * from hive.hive.thive; +insert into hive.hive.thive values (10, 20); +select * from hive.hive.thive; +purgedata hive.hive.thive; +purgedata hive.hive.thive; +select * from hive.hive.thive; + log; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c8a6187f/core/sql/regress/tools/runallsb ---------------------------------------------------------------------- diff --git a/core/sql/regress/tools/runallsb b/core/sql/regress/tools/runallsb index ac6f706..555ad93 100755 --- a/core/sql/regress/tools/runallsb +++ b/core/sql/regress/tools/runallsb @@ -41,7 +41,7 @@ export SQLMX_REGRESS=1 export SEABASE_REGRESS=2 -TEST_SUBDIRS="core compGeneral executor seabase fullstack2 charsets qat privs1 privs2 udr hive" +TEST_SUBDIRS="core compGeneral executor seabase hive fullstack2 charsets qat privs1 privs2 udr" # # Make sure we're running from the regress subdir
