Repository: trafodion Updated Branches: refs/heads/master 07e94f3ba -> fec690dea
Fixes for TRAFODION-3107, TRAFODION-3092, TRAFODION-3004 Project: http://git-wip-us.apache.org/repos/asf/trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/trafodion/commit/811d2bd1 Tree: http://git-wip-us.apache.org/repos/asf/trafodion/tree/811d2bd1 Diff: http://git-wip-us.apache.org/repos/asf/trafodion/diff/811d2bd1 Branch: refs/heads/master Commit: 811d2bd1a4c085299afb4886eb53480dcd036db9 Parents: bf8d8d6 Author: Sandhya Sundaresan <[email protected]> Authored: Fri Jun 15 18:46:04 2018 +0000 Committer: Sandhya Sundaresan <[email protected]> Committed: Fri Jun 15 18:46:04 2018 +0000 ---------------------------------------------------------------------- core/sql/cli/Context.cpp | 2 +- core/sql/cli/Statement.cpp | 14 +++++++++++++- core/sql/executor/ex_root.cpp | 4 ++-- core/sql/executor/sql_buffer.cpp | 2 +- core/sql/executor/sql_buffer.h | 4 ++-- core/sql/exp/ExpSqlTupp.h | 14 +++++++------- core/sql/generator/GenRelScan.cpp | 2 +- core/sql/generator/GenRelUpdate.cpp | 4 ++-- core/sql/generator/Generator.cpp | 10 +++++++++- core/sql/generator/Generator.h | 1 + core/sql/sqlcomp/DefaultConstants.h | 12 ++++++++---- core/sql/sqlcomp/nadefaults.cpp | 3 +++ 12 files changed, 50 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/811d2bd1/core/sql/cli/Context.cpp ---------------------------------------------------------------------- diff --git a/core/sql/cli/Context.cpp b/core/sql/cli/Context.cpp index 53d277e..46310d9 100644 --- a/core/sql/cli/Context.cpp +++ b/core/sql/cli/Context.cpp @@ -1274,7 +1274,7 @@ void ContextCli::closeAllCursors(enum CloseCursorType type, // scope. if (statement->getCliLevel() == getNumOfCliCalls()) { - statement->close(diagsArea_, inRollback); + statement->close(diagsArea_, inRollback); } // STATUSTRANSACTION slows down the response time // Browse access cursor that are started under a transaction http://git-wip-us.apache.org/repos/asf/trafodion/blob/811d2bd1/core/sql/cli/Statement.cpp ---------------------------------------------------------------------- diff --git a/core/sql/cli/Statement.cpp b/core/sql/cli/Statement.cpp index 1aeb708..4436e8c 100644 --- a/core/sql/cli/Statement.cpp +++ b/core/sql/cli/Statement.cpp @@ -3725,6 +3725,10 @@ RETCODE Statement::fetch(CliGlobals * cliGlobals, Descriptor * output_desc, { StmtDebug3("[END fetch] %p, result is %s, stmt state %s", this, RetcodeToString(ERROR), stmtState(getState())); + // In case there is a commit conflict, we need to reset the rowcount + // since none of the rows would have got committed. + diagsArea.setRowCount(0); + stmtStats_->getMasterStats()->setRowsAffected(0); return ERROR; } @@ -4891,7 +4895,15 @@ short Statement::commitTransaction(ComDiagsArea &diagsArea) // get current context and close all statements // started under the current transaction context_->closeAllCursors(ContextCli::CLOSE_ALL, ContextCli::CLOSE_CURR_XN); - + // Capture any errors that happened and return eg. transaction + // related errors that happen during + // Statement::close-> ExTransaction::commitTransaction that get called + // in ::closeAllCursors + if (diagsArea.mainSQLCODE() <0) + { + return ERROR; + } + // if transaction is still active(it may have been committed at // close cursor time if auto commit is on), commit it. if (context_->getTransaction()->xnInProgress()) http://git-wip-us.apache.org/repos/asf/trafodion/blob/811d2bd1/core/sql/executor/ex_root.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ex_root.cpp b/core/sql/executor/ex_root.cpp index 0fe945b..1a6fba5 100644 --- a/core/sql/executor/ex_root.cpp +++ b/core/sql/executor/ex_root.cpp @@ -387,7 +387,7 @@ ex_root_tcb::ex_root_tcb( tupp_descriptor *tp = new(glob->getSpace()) tupp_descriptor; char * dataPtr = (char *)glob->getSpace()->allocateMemory(root_tdb.pkeyLen_); - tp->init((short) root_tdb.pkeyLen_,0,dataPtr); + tp->init( root_tdb.pkeyLen_,0,dataPtr); workAtp_->getTupp(numTuples++) = tp; } else if (pkeyExpr()) @@ -398,7 +398,7 @@ ex_root_tcb::ex_root_tcb( tupp_descriptor *tp = new(glob->getSpace()) tupp_descriptor; char * dataPtr = (char *)glob->getSpace()->allocateMemory(root_tdb.pkeyLen_); - tp->init((short) root_tdb.pkeyLen_,0,dataPtr); + tp->init( root_tdb.pkeyLen_,0,dataPtr); pkeyAtp_->getTupp(2) = tp; } http://git-wip-us.apache.org/repos/asf/trafodion/blob/811d2bd1/core/sql/executor/sql_buffer.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/sql_buffer.cpp b/core/sql/executor/sql_buffer.cpp index 911ce11..8b8c166 100644 --- a/core/sql/executor/sql_buffer.cpp +++ b/core/sql/executor/sql_buffer.cpp @@ -1818,7 +1818,7 @@ tupp_descriptor *SqlBufferDense::add_tuple_desc(Lng32 tup_data_size) tdi = firstTupleDesc(); tupp_descriptor * td = tdi->tupleDesc(); - td->init((unsigned short)tup_data_size, + td->init(tup_data_size, 0, (char *)td + td_size); http://git-wip-us.apache.org/repos/asf/trafodion/blob/811d2bd1/core/sql/executor/sql_buffer.h ---------------------------------------------------------------------- diff --git a/core/sql/executor/sql_buffer.h b/core/sql/executor/sql_buffer.h index ddfa40f..976aa7e 100644 --- a/core/sql/executor/sql_buffer.h +++ b/core/sql/executor/sql_buffer.h @@ -110,12 +110,12 @@ public: void setNextTDIOffset(ULng32 tdiOffset) { - nextTDIOffset() = (short)tdiOffset; + nextTDIOffset() = tdiOffset; } void setPrevTDIOffset(ULng32 tdiOffset) { - prevTDIOffset() = (short)tdiOffset; + prevTDIOffset() = tdiOffset; } http://git-wip-us.apache.org/repos/asf/trafodion/blob/811d2bd1/core/sql/exp/ExpSqlTupp.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpSqlTupp.h b/core/sql/exp/ExpSqlTupp.h index 3b97626..f368d1a 100644 --- a/core/sql/exp/ExpSqlTupp.h +++ b/core/sql/exp/ExpSqlTupp.h @@ -142,20 +142,20 @@ friend class tupp; struct { - unsigned short nextTDIOffset_; - unsigned short prevTDIOffset_; + ULng32 nextTDIOffset_; + ULng32 prevTDIOffset_; } tdiOffset_; }; union { - Lng32 offset_; + Int64 offset_; char * tupleAddress_; }; protected: - unsigned short& nextTDIOffset() { return tdiOffset_.nextTDIOffset_; } - unsigned short& prevTDIOffset() { return tdiOffset_.prevTDIOffset_; } + ULng32& nextTDIOffset() { return tdiOffset_.nextTDIOffset_; } + ULng32& prevTDIOffset() { return tdiOffset_.prevTDIOffset_; } public: @@ -183,12 +183,12 @@ void resetCommFlags() inline char *getTupleAddress() const; - inline Lng32 getTupleOffset() const + inline Int64 getTupleOffset() const { return offset_; } - inline void setTupleOffset(Lng32 offset) + inline void setTupleOffset(Int64 offset) { offset_ = offset; } http://git-wip-us.apache.org/repos/asf/trafodion/blob/811d2bd1/core/sql/generator/GenRelScan.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelScan.cpp b/core/sql/generator/GenRelScan.cpp index 827b94c..4b1b397 100644 --- a/core/sql/generator/GenRelScan.cpp +++ b/core/sql/generator/GenRelScan.cpp @@ -3141,7 +3141,7 @@ short HbaseAccess::codeGen(Generator * generator) generator->setHBaseNumCacheRows(MAXOF(getEstRowsAccessed().getValue(), getMaxCardEst().getValue()), - hbpa, samplePercent()) ; + hbpa, hbaseRowSize,samplePercent()) ; generator->setHBaseCacheBlocks(hbaseRowSize, getEstRowsAccessed().getValue(),hbpa); generator->setHBaseSmallScanner(hbaseRowSize, http://git-wip-us.apache.org/repos/asf/trafodion/blob/811d2bd1/core/sql/generator/GenRelUpdate.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelUpdate.cpp b/core/sql/generator/GenRelUpdate.cpp index b7cc6bb..b8ae57f 100644 --- a/core/sql/generator/GenRelUpdate.cpp +++ b/core/sql/generator/GenRelUpdate.cpp @@ -1200,7 +1200,7 @@ short HbaseDelete::codeGen(Generator * generator) if (CmpCommon::getDefault(HBASE_CACHE_BLOCKS) != DF_OFF) hbpa->setCacheBlocks(TRUE); // estrowsaccessed is 0 for now, so cache size will be set to minimum - generator->setHBaseNumCacheRows(getEstRowsAccessed().getValue(), hbpa) ; + generator->setHBaseNumCacheRows(getEstRowsAccessed().getValue(), hbpa,rowIdAsciiRowLen) ; // create hdfsscan_tdb ComTdbHbaseAccess *hbasescan_tdb = new(space) @@ -2163,7 +2163,7 @@ short HbaseUpdate::codeGen(Generator * generator) if (CmpCommon::getDefault(HBASE_CACHE_BLOCKS) != DF_OFF) hbpa->setCacheBlocks(TRUE); // estrowsaccessed is 0 for now, so cache size will be set to minimum - generator->setHBaseNumCacheRows(getEstRowsAccessed().getValue(), hbpa) ; + generator->setHBaseNumCacheRows(getEstRowsAccessed().getValue(), hbpa,asciiRowLen) ; // create hdfsscan_tdb http://git-wip-us.apache.org/repos/asf/trafodion/blob/811d2bd1/core/sql/generator/Generator.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/Generator.cpp b/core/sql/generator/Generator.cpp index 20f799a..895816c 100644 --- a/core/sql/generator/Generator.cpp +++ b/core/sql/generator/Generator.cpp @@ -3291,8 +3291,11 @@ NABoolean Generator::considerDefragmentation( const ValueIdList & valIdList, void Generator::setHBaseNumCacheRows(double estRowsAccessed, ComTdbHbaseAccess::HbasePerfAttributes * hbpa, - Float32 samplePercent) + Int32 hbaseRowSize, + Float32 samplePercent + ) { + // compute the number of rows accessed per scan node instance and use it // to set HBase scan cache size (in units of number of rows). This cache // is in the HBase client, i.e. in the java side of @@ -3334,6 +3337,11 @@ void Generator::setHBaseNumCacheRows(double estRowsAccessed, } } + // Limit the scanner cache size to a fixed number if we are dealing with + // very wide rows eg rows with varchar(16MB) + Int32 maxRowSizeInCache = CmpCommon::getDefaultNumeric(TRAF_MAX_ROWSIZE_IN_CACHE)*1024*1024; + if (hbaseRowSize > maxRowSizeInCache) + cacheRows = 2; hbpa->setNumCacheRows(cacheRows); } http://git-wip-us.apache.org/repos/asf/trafodion/blob/811d2bd1/core/sql/generator/Generator.h ---------------------------------------------------------------------- diff --git a/core/sql/generator/Generator.h b/core/sql/generator/Generator.h index 8f2f50d..bf9e04b 100644 --- a/core/sql/generator/Generator.h +++ b/core/sql/generator/Generator.h @@ -1647,6 +1647,7 @@ public: void setHBaseNumCacheRows(double rowsAccessed, ComTdbHbaseAccess::HbasePerfAttributes * hbpa, + Int32 hbaseRowSize, Float32 samplePercent = 0.0); void setHBaseCacheBlocks(Int32 hbaseRowSize, double rowsAccessed, ComTdbHbaseAccess::HbasePerfAttributes * hbpa); http://git-wip-us.apache.org/repos/asf/trafodion/blob/811d2bd1/core/sql/sqlcomp/DefaultConstants.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h index 1a9304e..992f1ee 100644 --- a/core/sql/sqlcomp/DefaultConstants.h +++ b/core/sql/sqlcomp/DefaultConstants.h @@ -1962,7 +1962,6 @@ enum DefaultConstants TMUDF_LEAF_CARDINALITY, UDF_SUBQ_IN_AGGS_AND_GBYS, - USTAT_FETCHCOUNT_ACTIVE, SEMIJOIN_TO_INNERJOIN_INNER_ALLOWANCE, @@ -2875,8 +2874,7 @@ enum DefaultConstants HBASE_MIN_BYTES_PER_ESP_PARTITION, HBASE_MAX_ESPS, - - + // for testing setjmp/longjmp compiler logic MEMORY_LIMIT_CMPCTXT_UPPER_KB, @@ -3015,9 +3013,15 @@ enum DefaultConstants // In special cases, previous default value could be overridden. // Internal use only or use only under trafodion supervision. TRAF_MAX_CHARACTER_COL_LENGTH_OVERRIDE, - // set when metadata definition is to be read from hardcoded structs // and not from metadata. + + // set to limit the number of rows in scanner cache when we have very + // wide rows. If the rows are too large we may run into an OOM error + // since weuse HBASE_NUM_CACHE_ROWS_MIN(MAX) CQDs to calculate + // the number of rows + TRAF_MAX_ROWSIZE_IN_CACHE, + TRAF_BOOTSTRAP_MD_MODE, UDR_DEBUG_FLAGS, http://git-wip-us.apache.org/repos/asf/trafodion/blob/811d2bd1/core/sql/sqlcomp/nadefaults.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp index 3622db1..5146b32 100644 --- a/core/sql/sqlcomp/nadefaults.cpp +++ b/core/sql/sqlcomp/nadefaults.cpp @@ -2925,6 +2925,9 @@ XDDkwd__(SUBQUERY_UNNESTING, "ON"), // max size in bytes of a char or varchar column. Set to 16M DDui___(TRAF_MAX_CHARACTER_COL_LENGTH, MAX_CHAR_COL_LENGTH_IN_BYTES_STR), DDkwd__(TRAF_MAX_CHARACTER_COL_LENGTH_OVERRIDE, "OFF"), + // max size in MB of a row that canbe accomodated in scanner cache when + // using th the default scanner cache size. + DDint__(TRAF_MAX_ROWSIZE_IN_CACHE, "10"), DDkwd__(TRAF_MULTI_COL_FAM, "ON"),
