Repository: incubator-trafodion Updated Branches: refs/heads/master 530ad721b -> c58587a30
[TRAFODION-1655] RMS reports incorrect hbase io time and 0 for OperCpuTime for operators hosted in ESP Isolate the timer for Operator Cpu Time from HBase IO Timer. Statistics enabling flag was not set though statistics area was created and some of the metrics were collected. Operator Cpu time metrics collection requires statistics to be enabled. It is now enabled when the operators are hosted in ESPs. Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/127fb164 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/127fb164 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/127fb164 Branch: refs/heads/master Commit: 127fb16428de6138c45588537ebca26253d7bfa4 Parents: 80d6cff Author: selvaganesang <[email protected]> Authored: Thu Dec 3 18:45:56 2015 +0000 Committer: selvaganesang <[email protected]> Committed: Thu Dec 3 18:45:56 2015 +0000 ---------------------------------------------------------------------- core/sql/executor/ExHdfsScan.cpp | 4 +- core/sql/executor/ExStats.cpp | 16 +++++++- core/sql/executor/ExStats.h | 4 +- core/sql/executor/HBaseClient_JNI.cpp | 60 +++++++++++++++--------------- core/sql/executor/ex_split_bottom.cpp | 11 ++++++ 5 files changed, 59 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/127fb164/core/sql/executor/ExHdfsScan.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp index 71c0118..31fa587 100644 --- a/core/sql/executor/ExHdfsScan.cpp +++ b/core/sql/executor/ExHdfsScan.cpp @@ -598,7 +598,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work() "too many bites."); if (hdfsStats_) - hdfsStats_->getTimer().start(); + hdfsStats_->getHdfsTimer().start(); retcode = 0; @@ -650,7 +650,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work() ); if (hdfsStats_) - hdfsStats_->incMaxHdfsIOTime(hdfsStats_->getTimer().stop()); + hdfsStats_->incMaxHdfsIOTime(hdfsStats_->getHdfsTimer().stop()); if (retcode < 0) { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/127fb164/core/sql/executor/ExStats.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExStats.cpp b/core/sql/executor/ExStats.cpp index 2643443..2e544d8 100644 --- a/core/sql/executor/ExStats.cpp +++ b/core/sql/executor/ExStats.cpp @@ -1468,7 +1468,10 @@ NABoolean ExOperStats::operator==(ExOperStats * other) statType() == ExOperStats::BMO_STATS || statType() == ExOperStats::UDR_BASE_STATS || statType() == ExOperStats::REPLICATOR_STATS || - statType() == ExOperStats::REORG_STATS) + statType() == ExOperStats::REORG_STATS || + statType() == ExOperStats::HBASE_ACCESS_STATS || + statType() == ExOperStats::HDFSSCAN_STATS) + { if (getId()->tdbId_ == other->getId()->tdbId_) return TRUE; @@ -1950,6 +1953,12 @@ void ExFragRootOperStats::merge(ExOperStats * other) case BMO_STATS: merge((ExBMOStats *)other); break; + case HBASE_ACCESS_STATS: + merge((ExHbaseAccessStats *)other); + break; + case HDFSSCAN_STATS: + merge((ExHdfsScanStats *)other); + break; default: // do nothing - This type of stat has no merge data break; @@ -5403,7 +5412,7 @@ void ExMeasStats::merge(ExHbaseAccessStats* other) exeDp2Stats()->incAccessedDP2Rows(other->rowsAccessed()); exeDp2Stats()->incUsedDP2Rows(other->rowsUsed()); exeDp2Stats()->incDiskReads(other->hbaseCalls()); - exeDp2Stats()->incProcessBusyTime(other->getTimer().getTime()); + exeDp2Stats()->incProcessBusyTime(other->getHbaseTimer().getTime()); fsDp2MsgsStats() ->incMessageBytes(0); } @@ -6691,7 +6700,10 @@ NABoolean ExStatisticsArea::merge(ExOperStats * other, UInt16 statsMergeType) other->statType() != ExOperStats::BMO_STATS && other->statType() != ExOperStats::PERTABLE_STATS && other->statType() != ExOperStats::UDR_BASE_STATS && + other->statType() != ExOperStats::HBASE_ACCESS_STATS && + other->statType() != ExOperStats::HDFSSCAN_STATS && other->statType() != ExOperStats::REORG_STATS) + // Ignore stats and return as if merge is done return TRUE; while ((stat = getNext()) != NULL) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/127fb164/core/sql/executor/ExStats.h ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExStats.h b/core/sql/executor/ExStats.h index db1c46b..e27351b 100644 --- a/core/sql/executor/ExStats.h +++ b/core/sql/executor/ExStats.h @@ -2602,7 +2602,7 @@ NA_EIDPROC char * tableName() const {return tableName_;} NA_EIDPROC - ExTimeStats &getTimer() { return timer_; } + ExTimeStats &getHdfsTimer() { return timer_; } NA_EIDPROC inline void incBytesRead(Int64 bytesRead) {numBytesRead_ += bytesRead;} @@ -2702,7 +2702,7 @@ class ExHbaseAccessStats : public ExOperStats { char * tableName() const {return tableName_;} NA_EIDPROC - ExTimeStats &getTimer() { return timer_; } + ExTimeStats &getHbaseTimer() { return timer_; } NA_EIDPROC inline void incBytesRead(Int64 bytesRead) {numBytesRead_ += bytesRead;} http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/127fb164/core/sql/executor/HBaseClient_JNI.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/HBaseClient_JNI.cpp b/core/sql/executor/HBaseClient_JNI.cpp index e0712b2..f074b57 100644 --- a/core/sql/executor/HBaseClient_JNI.cpp +++ b/core/sql/executor/HBaseClient_JNI.cpp @@ -2048,14 +2048,14 @@ HBLC_RetCode HBulkLoadClient_JNI::addToHFile( short rowIDLen, HbaseStr &rowIDs, jshort j_rowIDLen = rowIDLen; if (hbs) - hbs->getTimer().start(); + hbs->getHbaseTimer().start(); tsRecentJMFromJNI = JavaMethods_[JM_ADD_TO_HFILE_DB].jm_full_name; jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_ADD_TO_HFILE_DB].methodID, j_rowIDLen, jRowIDs, jRows); if (hbs) { - hbs->incMaxHbaseIOTime(hbs->getTimer().stop()); + hbs->incMaxHbaseIOTime(hbs->getHbaseTimer().stop()); hbs->incHbaseCalls(); } jenv_->DeleteLocalRef(jRowIDs); @@ -2616,7 +2616,7 @@ HTableClient_JNI *HBaseClient_JNI::startGet(NAHeap *heap, const char* tableName, jlong j_ts = timestamp; if (hbs) - hbs->getTimer().start(); + hbs->getHbaseTimer().start(); tsRecentJMFromJNI = JavaMethods_[JM_START_GET].jm_full_name; jint jresult = jenv_->CallIntMethod(javaObj_, @@ -2624,7 +2624,7 @@ HTableClient_JNI *HBaseClient_JNI::startGet(NAHeap *heap, const char* tableName, (jlong)htc, js_tblName, (jboolean)useTRex, j_tid, jba_rowID, j_cols, j_ts); if (hbs) { - hbs->incMaxHbaseIOTime(hbs->getTimer().stop()); + hbs->incMaxHbaseIOTime(hbs->getHbaseTimer().stop()); hbs->incHbaseCalls(); } @@ -2727,7 +2727,7 @@ HTableClient_JNI *HBaseClient_JNI::startGets(NAHeap *heap, const char* tableName jlong j_ts = timestamp; jshort jRowIDLen = rowIDLen; if (hbs) - hbs->getTimer().start(); + hbs->getHbaseTimer().start(); jint jresult; if (rowIDs != NULL) { @@ -2744,7 +2744,7 @@ HTableClient_JNI *HBaseClient_JNI::startGets(NAHeap *heap, const char* tableName j_cols); } if (hbs) { - hbs->incMaxHbaseIOTime(hbs->getTimer().stop()); + hbs->incMaxHbaseIOTime(hbs->getHbaseTimer().stop()); hbs->incHbaseCalls(); } @@ -2938,14 +2938,14 @@ HBC_RetCode HBaseClient_JNI::insertRow(NAHeap *heap, const char *tableName, jboolean j_asyncOperation = asyncOperation; if (hbs) - hbs->getTimer().start(); + hbs->getHbaseTimer().start(); tsRecentJMFromJNI = JavaMethods_[JM_HBC_DIRECT_INSERT_ROW].jm_full_name; jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_HBC_DIRECT_INSERT_ROW].methodID, j_htc, js_tblName, j_useTRex, j_tid, jba_rowID, jRow, j_ts, j_checkAndPut, j_asyncOperation); if (hbs) { hbs->incHbaseCalls(); if (!asyncOperation) - hbs->incMaxHbaseIOTime(hbs->getTimer().stop()); + hbs->incMaxHbaseIOTime(hbs->getHbaseTimer().stop()); } if (jenv_->ExceptionCheck()) { getExceptionDetails(); @@ -3029,14 +3029,14 @@ HBC_RetCode HBaseClient_JNI::insertRows(NAHeap *heap, const char *tableName, jboolean j_asyncOperation = asyncOperation; if (hbs) - hbs->getTimer().start(); + hbs->getHbaseTimer().start(); tsRecentJMFromJNI = JavaMethods_[JM_HBC_DIRECT_INSERT_ROWS].jm_full_name; jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_HBC_DIRECT_INSERT_ROWS].methodID, j_htc, js_tblName, j_useTRex, j_tid, j_rowIDLen, jRowIDs, jRows, j_ts, j_af, j_asyncOperation); if (hbs) { hbs->incHbaseCalls(); if (!asyncOperation) - hbs->incMaxHbaseIOTime(hbs->getTimer().stop()); + hbs->incMaxHbaseIOTime(hbs->getHbaseTimer().stop()); } if (jenv_->ExceptionCheck()) { getExceptionDetails(); @@ -3141,7 +3141,7 @@ HBC_RetCode HBaseClient_JNI::checkAndUpdateRow(NAHeap *heap, const char *tableNa jboolean j_asyncOperation = asyncOperation; if (hbs) - hbs->getTimer().start(); + hbs->getHbaseTimer().start(); tsRecentJMFromJNI = JavaMethods_[JM_HBC_DIRECT_CHECKANDUPDATE_ROW].jm_full_name; jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_HBC_DIRECT_CHECKANDUPDATE_ROW].methodID, j_htc, js_tblName, j_useTRex, j_tid, jba_rowID, jRow, @@ -3149,7 +3149,7 @@ HBC_RetCode HBaseClient_JNI::checkAndUpdateRow(NAHeap *heap, const char *tableNa if (hbs) { hbs->incHbaseCalls(); if (!asyncOperation) - hbs->incMaxHbaseIOTime(hbs->getTimer().stop()); + hbs->incMaxHbaseIOTime(hbs->getHbaseTimer().stop()); } if (jenv_->ExceptionCheck()) { getExceptionDetails(); @@ -3234,7 +3234,7 @@ HBC_RetCode HBaseClient_JNI::deleteRow(NAHeap *heap, const char *tableName, jlong j_ts = timestamp; jboolean j_asyncOperation = asyncOperation; if (hbs) - hbs->getTimer().start(); + hbs->getHbaseTimer().start(); tsRecentJMFromJNI = JavaMethods_[JM_HBC_DELETE_ROW].jm_full_name; jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_HBC_DELETE_ROW].methodID, j_htc, js_tblName, j_useTRex, j_tid, jba_rowID, @@ -3242,7 +3242,7 @@ HBC_RetCode HBaseClient_JNI::deleteRow(NAHeap *heap, const char *tableName, if (hbs) { hbs->incHbaseCalls(); if (!asyncOperation) - hbs->incMaxHbaseIOTime(hbs->getTimer().stop()); + hbs->incMaxHbaseIOTime(hbs->getHbaseTimer().stop()); } if (jenv_->ExceptionCheck()) { getExceptionDetails(); @@ -3317,7 +3317,7 @@ HBC_RetCode HBaseClient_JNI::deleteRows(NAHeap *heap, const char *tableName, jlong j_ts = timestamp; jboolean j_asyncOperation = asyncOperation; if (hbs) - hbs->getTimer().start(); + hbs->getHbaseTimer().start(); tsRecentJMFromJNI = JavaMethods_[JM_HBC_DIRECT_DELETE_ROWS].jm_full_name; jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_HBC_DIRECT_DELETE_ROWS].methodID, j_htc, js_tblName, j_useTRex, j_tid, @@ -3325,7 +3325,7 @@ HBC_RetCode HBaseClient_JNI::deleteRows(NAHeap *heap, const char *tableName, if (hbs) { hbs->incHbaseCalls(); if (!asyncOperation) - hbs->incMaxHbaseIOTime(hbs->getTimer().stop()); + hbs->incMaxHbaseIOTime(hbs->getHbaseTimer().stop()); } if (jenv_->ExceptionCheck()) { getExceptionDetails(); @@ -3416,7 +3416,7 @@ HBC_RetCode HBaseClient_JNI::checkAndDeleteRow(NAHeap *heap, const char *tableNa jlong j_ts = timestamp; jboolean j_asyncOperation = asyncOperation; if (hbs) - hbs->getTimer().start(); + hbs->getHbaseTimer().start(); tsRecentJMFromJNI = JavaMethods_[JM_HBC_CHECKANDDELETE_ROW].jm_full_name; jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_HBC_CHECKANDDELETE_ROW].methodID, j_htc, js_tblName, j_useTRex, j_tid, jba_rowID, @@ -3424,7 +3424,7 @@ HBC_RetCode HBaseClient_JNI::checkAndDeleteRow(NAHeap *heap, const char *tableNa if (hbs) { hbs->incHbaseCalls(); if (!asyncOperation) - hbs->incMaxHbaseIOTime(hbs->getTimer().stop()); + hbs->incMaxHbaseIOTime(hbs->getHbaseTimer().stop()); } if (jenv_->ExceptionCheck()) { getExceptionDetails(); @@ -3759,7 +3759,7 @@ HTC_RetCode HTableClient_JNI::startScan(Int64 transID, const Text& startRowID, } if (hbs_) - hbs_->getTimer().start(); + hbs_->getHbaseTimer().start(); tsRecentJMFromJNI = JavaMethods_[JM_SCAN_OPEN].jm_full_name; jboolean jresult = jenv_->CallBooleanMethod( @@ -3773,7 +3773,7 @@ HTC_RetCode HTableClient_JNI::startScan(Int64 transID, const Text& startRowID, if (hbs_) { - hbs_->incMaxHbaseIOTime(hbs_->getTimer().stop()); + hbs_->incMaxHbaseIOTime(hbs_->getHbaseTimer().stop()); hbs_->incHbaseCalls(); } @@ -3846,14 +3846,14 @@ HTC_RetCode HTableClient_JNI::deleteRow(Int64 transID, HbaseStr &rowID, const LI jlong j_tid = transID; jlong j_ts = timestamp; if (hbs_) - hbs_->getTimer().start(); + hbs_->getHbaseTimer().start(); tsRecentJMFromJNI = JavaMethods_[JM_DELETE].jm_full_name; jboolean j_asyncOperation = FALSE; jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_DELETE].methodID, j_tid, jba_rowID, j_cols, j_ts, j_asyncOperation); if (hbs_) { - hbs_->incMaxHbaseIOTime(hbs_->getTimer().stop()); + hbs_->incMaxHbaseIOTime(hbs_->getHbaseTimer().stop()); hbs_->incHbaseCalls(); } @@ -4065,7 +4065,7 @@ HTC_RetCode HTableClient_JNI::coProcAggr(Int64 transID, jint j_ncr = numCacheRows; if (hbs_) - hbs_->getTimer().start(); + hbs_->getHbaseTimer().start(); tsRecentJMFromJNI = JavaMethods_[JM_COPROC_AGGR].jm_full_name; jarray jresult = (jarray)jenv_->CallObjectMethod(javaObj_, JavaMethods_[JM_COPROC_AGGR].methodID, j_tid, @@ -4073,7 +4073,7 @@ HTC_RetCode HTableClient_JNI::coProcAggr(Int64 transID, jba_colname, j_cb, j_ncr); if (hbs_) { - hbs_->incMaxHbaseIOTime(hbs_->getTimer().stop()); + hbs_->incMaxHbaseIOTime(hbs_->getHbaseTimer().stop()); hbs_->incHbaseCalls(); } @@ -5241,14 +5241,14 @@ HTC_RetCode HTableClient_JNI::fetchRows() } jlong jniObject = (jlong)this; if (hbs_) - hbs_->getTimer().start(); + hbs_->getHbaseTimer().start(); tsRecentJMFromJNI = JavaMethods_[JM_FETCH_ROWS].jm_full_name; jint jRowsReturned = jenv_->CallIntMethod(javaObj_, JavaMethods_[JM_FETCH_ROWS].methodID, jniObject); if (hbs_) { - hbs_->incMaxHbaseIOTime(hbs_->getTimer().stop()); + hbs_->incMaxHbaseIOTime(hbs_->getHbaseTimer().stop()); hbs_->incHbaseCalls(); } @@ -5380,7 +5380,7 @@ HTC_RetCode HTableClient_JNI::completeAsyncOperation(Int32 timeout, NABoolean *r if (jenv_->PushLocalFrame(jniHandleCapacity_) != 0) { getExceptionDetails(); if (hbs_) - hbs_->incMaxHbaseIOTime(hbs_->getTimer().stop()); + hbs_->incMaxHbaseIOTime(hbs_->getHbaseTimer().stop()); return HTC_ERROR_COMPLETEASYNCOPERATION_EXCEPTION; } jint jtimeout = timeout; @@ -5391,7 +5391,7 @@ HTC_RetCode HTableClient_JNI::completeAsyncOperation(Int32 timeout, NABoolean *r logError(CAT_SQL_HBASE, "HTableClient_JNI::completeAsyncOperation()", getLastError()); jenv_->PopLocalFrame(NULL); if (hbs_) - hbs_->incMaxHbaseIOTime(hbs_->getTimer().stop()); + hbs_->incMaxHbaseIOTime(hbs_->getHbaseTimer().stop()); return HTC_ERROR_COMPLETEASYNCOPERATION_EXCEPTION; } jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_COMPLETE_PUT].methodID, @@ -5402,7 +5402,7 @@ HTC_RetCode HTableClient_JNI::completeAsyncOperation(Int32 timeout, NABoolean *r logError(CAT_SQL_HBASE, "HTableClient_JNI::completeAsyncOperation()", getLastError()); jenv_->PopLocalFrame(NULL); if (hbs_) - hbs_->incMaxHbaseIOTime(hbs_->getTimer().stop()); + hbs_->incMaxHbaseIOTime(hbs_->getHbaseTimer().stop()); return HTC_ERROR_COMPLETEASYNCOPERATION_EXCEPTION; } if (jresult == false) { @@ -5410,7 +5410,7 @@ HTC_RetCode HTableClient_JNI::completeAsyncOperation(Int32 timeout, NABoolean *r return HTC_ERROR_ASYNC_OPERATION_NOT_COMPLETE; } if (hbs_) - hbs_->incMaxHbaseIOTime(hbs_->getTimer().stop()); + hbs_->incMaxHbaseIOTime(hbs_->getHbaseTimer().stop()); jboolean *returnArray = jenv_->GetBooleanArrayElements(jresultArray, NULL); for (int i = 0; i < resultArrayLen; i++) resultArray[i] = returnArray[i]; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/127fb164/core/sql/executor/ex_split_bottom.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ex_split_bottom.cpp b/core/sql/executor/ex_split_bottom.cpp index 7a0c2bb..8152417 100644 --- a/core/sql/executor/ex_split_bottom.cpp +++ b/core/sql/executor/ex_split_bottom.cpp @@ -115,6 +115,12 @@ ex_split_bottom_tcb * ex_split_bottom_tdb::buildESPTcbTree( // ExStatisticsArea. if (getCollectStats()) { + glob->setStatsEnabled(TRUE); +#ifdef _DEBUG + if (getenv("DISABLE_STATS")) + glob->setStatsEnabled(FALSE); +#endif + ExStatisticsArea* statsArea; StatsGlobals *statsGlobals = espInstanceDir->getStatsGlobals(); if (statsGlobals == NULL || getCollectStatsType() == ALL_STATS || @@ -123,6 +129,7 @@ ex_split_bottom_tcb * ex_split_bottom_tdb::buildESPTcbTree( statsArea = new(espInstanceDir->getLocalStatsHeap()) ExStatisticsArea(espInstanceDir->getLocalStatsHeap(), numOfParentInstances, getCollectStatsType()); + statsArea->setStatsEnabled(glob->statsEnabled()); glob->setStatsArea(statsArea); result->allocateStatsEntry(); } @@ -136,6 +143,7 @@ ex_split_bottom_tcb * ex_split_bottom_tdb::buildESPTcbTree( statsArea = new(espInstanceDir->getStatsHeap()) ExStatisticsArea(espInstanceDir->getStatsHeap(), numOfParentInstances, getCollectStatsType()); + statsArea->setStatsEnabled(glob->statsEnabled()); StmtStats *ss = glob->castToExEspStmtGlobals()->setStmtStats(); if ( ss != NULL) ss->setStatsArea(statsArea); @@ -147,6 +155,9 @@ ex_split_bottom_tcb * ex_split_bottom_tdb::buildESPTcbTree( savedPriority, savedStopMode); } } + else + glob->setStatsEnabled(FALSE); + if (cpuLimit_ > 0) {
