Repository: incubator-trafodion Updated Branches: refs/heads/master 8a01b5a8d -> 4639458b0
[TRAFODION-2733] BMO quota changes To ensure BMO quota assignment works even when the estimates are way off. Introduced a CQD BMO_MEMORY_ESTIMATE_OUTLIER_FACTOR. When a BMO memory estimate exceeds BMO_MEMORY_LIMIT_PER_NODE_IN_MB by this factor, the BMO memory estimate for the operator is reset to BMO_MEMORY_LIMIT_PER_NODE_IN_MB * BMO_MEMORY_ESTIMATE_OUTLIER_FACTOR. Then BMO memory estimate and the quota is assigned based on the revised estimate. Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/375eeeab Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/375eeeab Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/375eeeab Branch: refs/heads/master Commit: 375eeeab7a178348c5d3ac6e89e40ee8a06cb0d2 Parents: 19c7544 Author: selvaganesang <[email protected]> Authored: Wed Dec 6 19:50:04 2017 +0000 Committer: selvaganesang <[email protected]> Committed: Thu Dec 14 01:26:03 2017 +0000 ---------------------------------------------------------------------- core/sql/executor/ExExeUtilGetStats.cpp | 4 +- core/sql/generator/GenExplain.cpp | 12 +--- core/sql/generator/GenPreCode.cpp | 29 +++----- core/sql/generator/GenProbeCache.cpp | 10 +-- core/sql/generator/GenRelEnforcer.cpp | 6 +- core/sql/generator/GenRelGrby.cpp | 79 +++++++------------- core/sql/generator/GenRelJoin.cpp | 76 +++++--------------- core/sql/generator/GenRelMisc.cpp | 66 +++++------------ core/sql/generator/GenRelSequence.cpp | 38 +++++----- core/sql/generator/Generator.cpp | 104 +++++++++++++++++++++++---- core/sql/generator/Generator.h | 90 +++++++++++++++-------- core/sql/optimizer/RelEnforcer.h | 9 +-- core/sql/optimizer/RelExpr.cpp | 18 +++-- core/sql/optimizer/RelExpr.h | 12 ++-- core/sql/optimizer/RelFastTransport.cpp | 2 +- core/sql/optimizer/RelFastTransport.h | 2 +- core/sql/optimizer/RelGrby.h | 5 +- core/sql/optimizer/RelJoin.h | 5 +- core/sql/optimizer/RelProbeCache.h | 4 +- core/sql/optimizer/RelRoutine.cpp | 2 +- core/sql/optimizer/RelRoutine.h | 2 +- core/sql/optimizer/RelSequence.h | 4 +- core/sql/regress/hive/EXPECTED030 | 7 -- core/sql/regress/seabase/EXPECTED010 | 84 ---------------------- core/sql/regress/seabase/EXPECTED011 | 1 - core/sql/regress/seabase/EXPECTED016 | 7 -- core/sql/sqlcomp/DefaultConstants.h | 3 +- core/sql/sqlcomp/nadefaults.cpp | 1 + 28 files changed, 288 insertions(+), 394 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/executor/ExExeUtilGetStats.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtilGetStats.cpp b/core/sql/executor/ExExeUtilGetStats.cpp index 8f9a3cd..e0afe2d 100644 --- a/core/sql/executor/ExExeUtilGetStats.cpp +++ b/core/sql/executor/ExExeUtilGetStats.cpp @@ -3161,6 +3161,7 @@ short ExExeUtilGetRTSStatisticsTcb::work() case FORMAT_AND_RETURN_BMO_STATS_: { const char *ofMode; + Int32 dop = 1; for (; currStatsItemEntry_ < maxBMOStatsItems_; currStatsItemEntry_++) { i = (short)currStatsItemEntry_; @@ -3186,6 +3187,7 @@ short ExExeUtilGetRTSStatisticsTcb::work() sprintf(&statsBuf_[strlen(statsBuf_)], "%20s", ofMode); break; case SQLSTATS_DOP: + dop = (Int32)bmoStatsItems_[i].int64_value; sprintf(&statsBuf_[strlen(statsBuf_)], "%10ld", bmoStatsItems_[i].int64_value); break; case SQLSTATS_TOPN: @@ -3234,7 +3236,7 @@ short ExExeUtilGetRTSStatisticsTcb::work() sprintf(&statsBuf_[strlen(statsBuf_)], "%20s", Int64Val); break; case SQLSTATS_BMO_EST_MEMORY: - sprintf(formattedFloatVal, "%0.2lf", bmoStatsItems_[i].double_value); + sprintf(formattedFloatVal, "%.6g", bmoStatsItems_[i].double_value * dop); str_sprintf(&statsBuf_[strlen(statsBuf_)], "%-20s", formattedFloatVal); break; case SQLSTATS_BMO_SPACE_BUFFER_SIZE: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/generator/GenExplain.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenExplain.cpp b/core/sql/generator/GenExplain.cpp index 17a8c7a..6626189 100644 --- a/core/sql/generator/GenExplain.cpp +++ b/core/sql/generator/GenExplain.cpp @@ -457,14 +457,14 @@ RelExpr::addExplainInfo(ComTdb * tdb, if ( reportMemoryEst == TRUE ) { if (nodeType == ComTdb::ex_HASH_GRBY || nodeType == ComTdb::ex_HASHJ || nodeType == ComTdb::ex_SORT) { - double memUsage = getEstimatedRunTimeMemoryUsage(FALSE).value()/1024; + double memUsage = tdb->getEstimatedMemoryUsage(); if ( memUsage > 0 ) { sprintf(buf, "est_memory_per_instance: %.3f KB ", memUsage); explainTuple->setDescription(buf); } } else { - double memUsage = getEstimatedRunTimeMemoryUsage(TRUE).value()/1024; + double memUsage = getEstimatedRunTimeMemoryUsage(generator, TRUE).value()/1024; if ( memUsage > 0 ) { sprintf(buf, "est_memory_per_node: %.3f KB ", memUsage); explainTuple->setDescription(buf); @@ -1442,14 +1442,6 @@ RelRoot::addSpecificExplainInfo(ExplainTupleMaster *explainTuple, statement += maxMaxCard; statement += " "; - double total_overflow_size = generator->getTotalOverflowMemory(); - statement += "total_overflow_size: "; - - char ovSizeVal[1024]; - sprintf(ovSizeVal, "%.2lf", total_overflow_size/1024); - statement += ovSizeVal; - statement += " KB "; - FragmentDir *fragDir = generator->getFragmentDir(); for (CollIndex i = 0; i < fragDir->entries(); i++) { if (fragDir->getPartitioningFunction(i) != NULL && http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/generator/GenPreCode.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenPreCode.cpp b/core/sql/generator/GenPreCode.cpp index 40baf03..f616852 100644 --- a/core/sql/generator/GenPreCode.cpp +++ b/core/sql/generator/GenPreCode.cpp @@ -1910,10 +1910,8 @@ RelExpr * RelRoot::preCodeGen(Generator * generator, if (generator->getBindWA()->getUdrStoiList().entries () > 0) generator->setAqrEnabled(FALSE); - // Reset the accumulated # of BMOs and memory usages in - // the generator + // Reset the accumulated # of BMOs for the fragment prevNumBMOs = generator->replaceNumBMOs(0); - prevBMOsMemoryUsage = generator->replaceBMOsMemoryUsage(0); } // true root @@ -2138,8 +2136,6 @@ RelExpr * RelRoot::preCodeGen(Generator * generator, // Remember # of BMOs that children's preCodeGen found for my fragment. setNumBMOs( generator->replaceNumBMOs(prevNumBMOs) ); - setBMOsMemoryUsage( generator->replaceBMOsMemoryUsage(prevBMOsMemoryUsage) ); - // Compute the total available memory quota for BMOs NADefaults &defs = ActiveSchemaDB()->getDefaults(); @@ -2234,6 +2230,8 @@ RelExpr * RelRoot::preCodeGen(Generator * generator, setHdfsAccess(generator->hdfsAccess()); + generator->finetuneBMOEstimates(); + markAsPreCodeGenned(); #ifdef _DEBUG @@ -3729,7 +3727,7 @@ RelExpr * HashJoin::preCodeGen(Generator * generator, generator->incrNumBMOs(); if ((ActiveSchemaDB()->getDefaults()).getAsDouble(BMO_MEMORY_LIMIT_PER_NODE_IN_MB) > 0) - generator->incrBMOsMemory(getEstimatedRunTimeMemoryUsage(TRUE)); + generator->incrBMOsMemory(getEstimatedRunTimeMemoryUsage(generator, TRUE)); // store the transformed predicates back into the hash join node @@ -5985,7 +5983,7 @@ RelExpr * GroupByAgg::preCodeGen(Generator * generator, generator->incrNumBMOs(); if ((ActiveSchemaDB()->getDefaults()).getAsDouble(BMO_MEMORY_LIMIT_PER_NODE_IN_MB) > 0) - generator->incrBMOsMemory(getEstimatedRunTimeMemoryUsage(TRUE)); + generator->incrBMOsMemory(getEstimatedRunTimeMemoryUsage(generator, TRUE)); } @@ -6624,7 +6622,7 @@ RelExpr * Sort::preCodeGen(Generator * generator, generator->incrNumBMOs(); if ((ActiveSchemaDB()->getDefaults()).getAsDouble(BMO_MEMORY_LIMIT_PER_NODE_IN_MB) > 0) - generator->incrBMOsMemory(getEstimatedRunTimeMemoryUsage(TRUE)); + generator->incrBMOsMemory(getEstimatedRunTimeMemoryUsage(generator, TRUE)); } } @@ -6720,15 +6718,14 @@ RelExpr *ProbeCache::preCodeGen(Generator * generator, (availableValues, getGroupAttr()->getCharacteristicInputs()); - /* +/* TBD - maybe ProbeCache as BMO memory participant?? if(CmpCommon::getDefault(PROBE_CACHE_MEMORY_QUOTA_SYSTEM) != DF_OFF) generator->incrNumBMOs(); - */ if ((ActiveSchemaDB()->getDefaults()).getAsDouble(BMO_MEMORY_LIMIT_PER_NODE_IN_MB) > 0) - generator->incrNBMOsMemoryPerNode(getEstimatedRunTimeMemoryUsage(TRUE)); - + generator->incrNBMOsMemoryPerNode(getEstimatedRunTimeMemoryUsage(generator, TRUE)); +*/ markAsPreCodeGenned(); return this; @@ -6906,7 +6903,6 @@ RelExpr * Exchange::preCodeGen(Generator * generator, NABoolean inputOltMsgOpt = generator->oltOptInfo()->oltMsgOpt(); unsigned short prevNumBMOs = generator->replaceNumBMOs(0); - CostScalar prevBMOsMemoryUsage = generator->replaceBMOsMemoryUsage(0); // These are used to fix solution 10-071204-9253 and for // solution 10-100310-8659. @@ -7069,7 +7065,6 @@ RelExpr * Exchange::preCodeGen(Generator * generator, generator->setHalloweenESPonLHS(halloweenESPonLHS); setNumBMOs( generator->replaceNumBMOs(prevNumBMOs) ); - setBMOsMemoryUsage( generator->replaceBMOsMemoryUsage(prevBMOsMemoryUsage) ); if (! child(0).getPtr()) return NULL; @@ -7201,9 +7196,8 @@ RelExpr * Exchange::preCodeGen(Generator * generator, { result = child(0).getPtr(); - // transfer the # of BMOs and their memory usages to generator as + // transfer the # of BMOs to generator as // this exchange node is to be discarded. - generator->incrBMOsMemoryPerFrag(getBMOsMemoryUsage()); generator->incrNumBMOsPerFrag(getNumBMOs()); } @@ -7231,8 +7225,7 @@ RelExpr * Exchange::preCodeGen(Generator * generator, } // isEspExchange() && !eliminateThisExchange - if ((ActiveSchemaDB()->getDefaults()).getAsDouble(BMO_MEMORY_LIMIT_PER_NODE_IN_MB) > 0) - generator->incrNBMOsMemoryPerNode(getEstimatedRunTimeMemoryUsage(TRUE)); + return result; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/generator/GenProbeCache.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenProbeCache.cpp b/core/sql/generator/GenProbeCache.cpp index e6c6d41..17a6dce 100644 --- a/core/sql/generator/GenProbeCache.cpp +++ b/core/sql/generator/GenProbeCache.cpp @@ -428,9 +428,9 @@ short ProbeCache::codeGen(Generator *generator) probeCacheTdb->getQueueResizeFactor()); } - double probeCacheMemEst = getEstimatedRunTimeMemoryUsage(probeCacheTdb); + double probeCacheMemEst = getEstimatedRunTimeMemoryUsage(generator, probeCacheTdb); generator->addToTotalEstimatedMemory(probeCacheMemEst); - Lng32 pcMemEstInKBPerNode = getEstimatedRunTimeMemoryUsage(TRUE).value() / 1024; + Lng32 pcMemEstInKBPerNode = getEstimatedRunTimeMemoryUsage(generator, TRUE).value() / 1024; if(!generator->explainDisabled()) { generator->setExplainTuple( addExplainInfo(probeCacheTdb, childExplainTuple, 0, generator)); @@ -443,7 +443,7 @@ short ProbeCache::codeGen(Generator *generator) return 0; } -CostScalar ProbeCache::getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStreams) +CostScalar ProbeCache::getEstimatedRunTimeMemoryUsage(Generator *generator, NABoolean perNode, Lng32 *numStreams) { const Lng32 probeSize = getGroupAttr()->getCharacteristicInputs().getRowLength(); @@ -488,12 +488,12 @@ CostScalar ProbeCache::getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 * return totalMemory; } -double ProbeCache::getEstimatedRunTimeMemoryUsage(ComTdb * tdb) +double ProbeCache::getEstimatedRunTimeMemoryUsage(Generator *generator, ComTdb * tdb) { // tdb is ignored for ProbeCache because this operator // does not participate in the BMO quota system. Lng32 numOfStreams = 1; - CostScalar totalMemory = getEstimatedRunTimeMemoryUsage(FALSE, &numOfStreams); + CostScalar totalMemory = getEstimatedRunTimeMemoryUsage(generator, FALSE, &numOfStreams); totalMemory = totalMemory * numOfStreams ; return totalMemory.value(); } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/generator/GenRelEnforcer.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelEnforcer.cpp b/core/sql/generator/GenRelEnforcer.cpp index 197aadb..bb557a1 100644 --- a/core/sql/generator/GenRelEnforcer.cpp +++ b/core/sql/generator/GenRelEnforcer.cpp @@ -1310,7 +1310,7 @@ ExpTupleDesc::TupleDataFormat Exchange::determineInternalFormat( const ValueIdLi considerBufferDefrag); } -CostScalar Exchange::getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStreams) +CostScalar Exchange::getEstimatedRunTimeMemoryUsage(Generator *generator, NABoolean perNode, Lng32 *numStreams) { ////////////////////////////////////// // compute the buffer length (for both @@ -1384,10 +1384,10 @@ CostScalar Exchange::getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *nu return memoryRequired; } -double Exchange::getEstimatedRunTimeMemoryUsage(ComTdb * tdb) +double Exchange::getEstimatedRunTimeMemoryUsage(Generator *generator, ComTdb * tdb) { Lng32 numOfStreams = 1; - CostScalar totalMemory = getEstimatedRunTimeMemoryUsage(FALSE, &numOfStreams); + CostScalar totalMemory = getEstimatedRunTimeMemoryUsage(generator, FALSE, &numOfStreams); totalMemory = totalMemory * numOfStreams ; return totalMemory.value(); } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/generator/GenRelGrby.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelGrby.cpp b/core/sql/generator/GenRelGrby.cpp index 4920a35..0ab7e2e 100644 --- a/core/sql/generator/GenRelGrby.cpp +++ b/core/sql/generator/GenRelGrby.cpp @@ -1474,7 +1474,7 @@ short HashGroupBy::codeGen(Generator * generator) { double memQuota = 0; Lng32 numStreams; double memQuotaRatio; - double bmoMemoryUsagePerNode = getEstimatedRunTimeMemoryUsage(TRUE, &numStreams).value(); + double bmoMemoryUsagePerNode = generator->getEstMemPerNode(getKey() ,numStreams); if(isPartialGroupBy) { // The Quota system does not apply to Partial GroupBy @@ -1536,16 +1536,12 @@ short HashGroupBy::codeGen(Generator * generator) { } - generator->addToTotalOverflowMemory( - getEstimatedRunTimeOverflowSize(memQuota) - ); - // For debugging overflow only (default is zero == not used). hashGrbyTdb-> setForceOverflowEvery((UInt16)(ActiveSchemaDB()->getDefaults()). getAsULong(EXE_TEST_HASH_FORCE_OVERFLOW_EVERY)); - double hashGBMemEst = getEstimatedRunTimeMemoryUsage(hashGrbyTdb); + double hashGBMemEst = generator->getEstMemPerInst(getKey()); hashGrbyTdb->setEstimatedMemoryUsage(hashGBMemEst / 1024); generator->addToTotalEstimatedMemory(hashGBMemEst); @@ -1620,22 +1616,26 @@ ExpTupleDesc::TupleDataFormat HashGroupBy::determineInternalFormat( const ValueI } -CostScalar HashGroupBy::getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStreams) +CostScalar HashGroupBy::getEstimatedRunTimeMemoryUsage(Generator *generator, NABoolean perNode, Lng32 *numStreams) { GroupAttributes * childGroupAttr = child(0).getGroupAttr(); const CostScalar childRecordSize = childGroupAttr->getCharacteristicOutputs().getRowLength(); const CostScalar childRowCount = getEstRowsUsed(); // the number of + //TODO: Line below dumps core at times + //const CostScalar maxCard = childGroupAttr->getResultMaxCardinalityForEmptyInput(); + const CostScalar maxCard = 0; + // distinct rows groupped // Each record also uses a header (HashRow) in memory (8 bytes for 32bit). // Hash tables also take memory -- they are about %50 longer than the // number of entries. const ULng32 memOverheadPerRecord = sizeof(HashRow) + sizeof(HashTableHeader) * 3 / 2 ; - + CostScalar estMemPerNode; + CostScalar estMemPerInst; // totalHashTableMemory is for all CPUs at this point of time. CostScalar totalHashTableMemory = childRowCount * (childRecordSize + memOverheadPerRecord); - Lng32 numOfStreams = 1; const PhysicalProperty* const phyProp = getPhysicalProperty(); if (phyProp) @@ -1647,51 +1647,22 @@ CostScalar HashGroupBy::getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 } if (numStreams != NULL) *numStreams = numOfStreams; - if (perNode) - totalHashTableMemory /= MINOF(MAXOF(((NAClusterInfoLinux*)gpClusterInfo)->getTotalNumberOfCPUs(), 1), numOfStreams); - else - totalHashTableMemory /= numOfStreams; - return totalHashTableMemory; -} - -double HashGroupBy::getEstimatedRunTimeMemoryUsage(ComTdb * tdb) -{ - Lng32 numOfStreams = 1; - CostScalar totalHashTableMemory = getEstimatedRunTimeMemoryUsage(FALSE, &numOfStreams); - totalHashTableMemory *= numOfStreams ; - return totalHashTableMemory.value(); -} - -double HashGroupBy::getEstimatedRunTimeOverflowSize(double memoryQuotaMB) -{ - - if ( memoryQuotaMB > 0 ) { - - CostScalar memoryUsage = - getEstimatedRunTimeMemoryUsage(TRUE /*per CPU*/); - - double delta = memoryUsage.getValue() - memoryQuotaMB * COM_ONE_MEG ; - - if ( delta > 0 ) { - const PhysicalProperty* const phyProp = getPhysicalProperty(); - Lng32 pipelines = 1; - - if (phyProp) - { - PartitioningFunction * partFunc = - phyProp -> getPartitioningFunction() ; - - if ( partFunc ) - pipelines = partFunc -> getCountOfPartitions(); - } - - - return delta * pipelines; - } - } - - return 0; - + estMemPerNode = totalHashTableMemory / MINOF(MAXOF(gpClusterInfo->getTotalNumberOfCPUs(), 1), numOfStreams); + estMemPerInst = totalHashTableMemory / numOfStreams; + NABoolean isPartialGroupBy = (isAPartialGroupByNonLeaf() || + isAPartialGroupByLeaf()); + if (isPartialGroupBy) + { + estMemPerNode = 1024; + estMemPerInst = 1024; + } + OperBMOQuota *operBMOQuota = new (generator->wHeap()) OperBMOQuota(getKey(), numOfStreams, + estMemPerNode, estMemPerInst, childRowCount, maxCard); + generator->getBMOQuotaMap()->insert(operBMOQuota); + if (perNode) + return estMemPerNode; + else + return estMemPerInst; } ///////////////////////////////////////////////////////// http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/generator/GenRelJoin.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelJoin.cpp b/core/sql/generator/GenRelJoin.cpp index 245b838..cc97b1d 100644 --- a/core/sql/generator/GenRelJoin.cpp +++ b/core/sql/generator/GenRelJoin.cpp @@ -1784,7 +1784,7 @@ short HashJoin::codeGen(Generator * generator) { double memQuota = 0; double memQuotaRatio; Lng32 numStreams; - double bmoMemoryUsagePerNode = getEstimatedRunTimeMemoryUsage(TRUE, &numStreams).value(); + double bmoMemoryUsagePerNode = generator->getEstMemPerNode(getKey(), numStreams); if (mmu != 0) { memQuota = mmu; hashj_tdb->setMemoryQuotaMB(mmu); @@ -1825,11 +1825,7 @@ short HashJoin::codeGen(Generator * generator) { if (beforeJoinPredOnOuterOnly()) hashj_tdb->setBeforePredOnOuterOnly(); - generator->addToTotalOverflowMemory( - getEstimatedRunTimeOverflowSize(memQuota) - ); - - double hjMemEst = getEstimatedRunTimeMemoryUsage(hashj_tdb); + double hjMemEst = generator->getEstMemPerInst(getKey()); hashj_tdb->setEstimatedMemoryUsage(hjMemEst / 1024); generator->addToTotalEstimatedMemory(hjMemEst); @@ -2059,17 +2055,23 @@ ExpTupleDesc::TupleDataFormat HashJoin::determineInternalFormat( const ValueIdLi } -CostScalar HashJoin::getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStreams) +CostScalar HashJoin::getEstimatedRunTimeMemoryUsage(Generator *generator, NABoolean perNode, Lng32 *numStreams) { GroupAttributes * childGroupAttr = child(1).getGroupAttr(); const CostScalar childRecordSize = childGroupAttr->getCharacteristicOutputs().getRowLength(); const CostScalar childRowCount = child(1).getPtr()->getEstRowsUsed(); + //TODO: Line below dumps core at times + //const CostScalar maxCard = childGroupAttr->getResultMaxCardinalityForEmptyInput(); + const CostScalar maxCard = 0; // Each record also uses a header (HashRow) in memory (8 bytes for 32bit). // Hash tables also take memory -- they are about %50 longer than the // number of entries. const ULng32 memOverheadPerRecord = sizeof(HashRow) + sizeof(HashTableHeader) * 3 / 2 ; + CostScalar estMemPerNode; + CostScalar estMemPerInst; + CostScalar totalHashTableMemory = childRowCount * (childRecordSize + memOverheadPerRecord); // one buffer for the outer table @@ -2089,57 +2091,15 @@ CostScalar HashJoin::getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *nu } if (numStreams != NULL) *numStreams = numOfStreams; - if (perNode) - totalHashTableMemory /= MINOF(MAXOF(((NAClusterInfoLinux*)gpClusterInfo)->getTotalNumberOfCPUs(), 1), numOfStreams); + estMemPerNode = totalHashTableMemory / MINOF(MAXOF(gpClusterInfo->getTotalNumberOfCPUs(), 1), numOfStreams); + estMemPerInst = totalHashTableMemory / numOfStreams; + OperBMOQuota *operBMOQuota = new (generator->wHeap()) OperBMOQuota(getKey(), numOfStreams, + estMemPerNode, estMemPerInst, childRowCount, maxCard); + generator->getBMOQuotaMap()->insert(operBMOQuota); + if (perNode) + return estMemPerNode; else - totalHashTableMemory /= numOfStreams; - return totalHashTableMemory; -} - -double HashJoin::getEstimatedRunTimeMemoryUsage(ComTdb * tdb) -{ - Lng32 numOfStreams = 1; - CostScalar totalHashTableMemory = getEstimatedRunTimeMemoryUsage(FALSE, &numOfStreams); - totalHashTableMemory *= numOfStreams ; - return totalHashTableMemory.value(); -} - -double HashJoin::getEstimatedRunTimeOverflowSize(double memoryQuotaMB) -{ - // Setup overflow size for join with formula ov = ((s0-m)/s0)*(s0+s1), where - // s0 = size of child0, s1 = size of child1 and m the memory quota for NJ - // - if ( memoryQuotaMB > 0 ) { - - GroupAttributes * c0 = child(0).getGroupAttr(); - double c0RLen = c0->getCharacteristicOutputs().getRowLength(); - double c0Rows = (child(0).getPtr()->getEstRowsUsed()).getValue(); - - GroupAttributes * c1 = child(1).getGroupAttr(); - double c1RLen = c1->getCharacteristicOutputs().getRowLength(); - double c1Rows = (child(1).getPtr()->getEstRowsUsed()).getValue(); - - double s0 = c0RLen * c0Rows; - double s1 = c1RLen * c1Rows; - - Lng32 pipelines = 1; - const PhysicalProperty* const phyProp = getPhysicalProperty() ; - if (phyProp) - { - PartitioningFunction * partFunc = phyProp -> getPartitioningFunction() ; - if ( partFunc ) - pipelines = partFunc->getCountOfPartitions(); - } - - double delta = s1 / pipelines - memoryQuotaMB * COM_ONE_MEG ; - if ( delta > 0 ) { - double ov = ((delta / s1) * (s0 + s1)) * pipelines; - return ov; - } - - } - - return 0; + return estMemPerInst; } // NABoolean HashJoin::canUseUniqueHashJoin() @@ -3072,7 +3032,7 @@ short MergeJoin::codeGen(Generator * generator) UInt16 quotaMB = 0; Lng32 numStreams; double memQuotaRatio; - double bmoMemoryUsage = getEstimatedRunTimeMemoryUsage(TRUE, &numStreams).value(); + double bmoMemoryUsage = generator->getEstMemPerNode(getKey(), numStreams); NADefaults &defs = ActiveSchemaDB()->getDefaults(); if ( CmpCommon::getDefaultLong(MJ_BMO_QUOTA_PERCENT) != 0) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/generator/GenRelMisc.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelMisc.cpp b/core/sql/generator/GenRelMisc.cpp index db46831..e8ed57d 100644 --- a/core/sql/generator/GenRelMisc.cpp +++ b/core/sql/generator/GenRelMisc.cpp @@ -2912,7 +2912,7 @@ short RelRoot::codeGen(Generator * generator) compilerStatsInfo->collectStatsType() = generator->collectStatsType(); compilerStatsInfo->udr() = noOfUdrs; compilerStatsInfo->ofMode() = generator->getOverflowMode(); - compilerStatsInfo->ofSize() = generator->getTotalOverflowMemory(); + compilerStatsInfo->ofSize() = 0; compilerStatsInfo->bmo() = generator->getTotalNumBMOs(); compilerStatsInfo->queryType() = (Int16)root_tdb->getQueryType(); compilerStatsInfo->subqueryType() = (Int16)root_tdb->getSubqueryType(); @@ -3061,7 +3061,7 @@ short Sort::generateTdb(Generator * generator, short memoryQuotaMB = 0; double memoryQuotaRatio; Lng32 numStreams; - double bmoMemoryUsagePerNode = getEstimatedRunTimeMemoryUsage(TRUE, &numStreams).value(); + double bmoMemoryUsagePerNode = generator->getEstMemPerNode(getKey(), numStreams); if (CmpCommon::getDefault(SORT_MEMORY_QUOTA_SYSTEM) != DF_OFF) { @@ -3172,13 +3172,10 @@ short Sort::generateTdb(Generator * generator, generator->initTdbFields(sort_tdb); - double sortMemEst = getEstimatedRunTimeMemoryUsage(sort_tdb); + double sortMemEst = generator->getEstMemPerInst(getKey()); sort_tdb->setEstimatedMemoryUsage(sortMemEst / 1024); generator->addToTotalEstimatedMemory(sortMemEst); - generator->addToTotalOverflowMemory( - getEstimatedRunTimeOverflowSize(memoryQuotaMB)); - if (sortPrefixKeyLen > 0) ((ComTdbSort *)sort_tdb)->setPartialSort(TRUE); // do partial sort @@ -3853,35 +3850,7 @@ short SortFromTop::codeGen(Generator * generator) return rc; } -double Sort::getEstimatedRunTimeOverflowSize(double memoryQuotaMB) -{ - if ( memoryQuotaMB > 0 ) { - CostScalar memoryUsage = getEstimatedRunTimeMemoryUsage(TRUE /*per CPU*/); - - double delta = memoryUsage.getValue() - memoryQuotaMB * COM_ONE_MEG ; - - if ( delta > 0 ) { - - const PhysicalProperty* const phyProp = getPhysicalProperty(); - Lng32 pipelines = 1; - - if (phyProp != NULL) - { - PartitioningFunction * partFunc = - phyProp -> getPartitioningFunction() ; - - if ( partFunc ) - pipelines = partFunc->getCountOfPartitions(); - } - - return delta * pipelines; - } - } - - return 0; -} - -CostScalar Sort::getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStreams) +CostScalar Sort::getEstimatedRunTimeMemoryUsage(Generator *generator, NABoolean perNode, Lng32 *numStreams) { GroupAttributes * childGroupAttr = child(0).getGroupAttr(); Lng32 childRecordSize = @@ -3893,6 +3862,12 @@ CostScalar Sort::getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStr else rowsUsed = getEstRowsUsed(); CostScalar totalMemory = rowsUsed * childRecordSize; + CostScalar estMemPerNode; + CostScalar estMemPerInst; + + //TODO: Line below dumps core at times + //const CostScalar maxCard = childGroupAttr->getResultMaxCardinalityForEmptyInput(); + const CostScalar maxCard = 0; Lng32 numOfStreams = 1; const PhysicalProperty* const phyProp = getPhysicalProperty(); @@ -3905,22 +3880,17 @@ CostScalar Sort::getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStr } if (numStreams != NULL) *numStreams = numOfStreams; - if (perNode) - totalMemory /= MINOF(MAXOF(((NAClusterInfoLinux*)gpClusterInfo)->getTotalNumberOfCPUs(), 1), numOfStreams); + estMemPerNode = totalMemory / MINOF(MAXOF(gpClusterInfo->getTotalNumberOfCPUs(), 1), numOfStreams); + estMemPerInst = totalMemory / numOfStreams; + OperBMOQuota *operBMOQuota = new (generator->wHeap()) OperBMOQuota(getKey(), numOfStreams, + estMemPerNode, estMemPerInst, rowsUsed, maxCard); + generator->getBMOQuotaMap()->insert(operBMOQuota); + if (perNode) + return estMemPerNode; else - totalMemory /= numOfStreams; - return totalMemory; + return estMemPerInst; } -double Sort::getEstimatedRunTimeMemoryUsage(ComTdb * tdb) -{ - Lng32 numOfStreams = 1; - CostScalar totalMemory = getEstimatedRunTimeMemoryUsage(FALSE, &numOfStreams); - totalMemory = totalMemory * numOfStreams ; - return totalMemory.value(); -} - - ///////////////////////////////////////////////////////// // // Tuple::codeGen() http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/generator/GenRelSequence.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelSequence.cpp b/core/sql/generator/GenRelSequence.cpp index 39a713e..41bfc20 100644 --- a/core/sql/generator/GenRelSequence.cpp +++ b/core/sql/generator/GenRelSequence.cpp @@ -778,10 +778,10 @@ RelExpr * PhysSequence::preCodeGen(Generator * generator, if ((ActiveSchemaDB()->getDefaults()). getAsDouble(BMO_MEMORY_LIMIT_PER_NODE_IN_MB) > 0) - generator->incrBMOsMemory(getEstimatedRunTimeMemoryUsage(TRUE)); + generator->incrBMOsMemory(getEstimatedRunTimeMemoryUsage(generator, TRUE)); } else - generator->incrNBMOsMemoryPerNode(getEstimatedRunTimeMemoryUsage(TRUE)); + generator->incrNBMOsMemoryPerNode(getEstimatedRunTimeMemoryUsage(generator, TRUE)); markAsPreCodeGenned(); @@ -1074,7 +1074,7 @@ PhysSequence::codeGen(Generator *generator) // update the estimated value of HistoryRowLength with actual value //setEstHistoryRowLength(historyIds.getRowLength()); - double sequenceMemEst = getEstimatedRunTimeMemoryUsage(sequenceTdb); + double sequenceMemEst = generator->getEstMemPerInst(getKey()); generator->addToTotalEstimatedMemory(sequenceMemEst); if(!generator->explainDisabled()) { @@ -1104,7 +1104,7 @@ PhysSequence::codeGen(Generator *generator) UInt16 mmu = (UInt16)(defs.getAsDouble(EXE_MEM_LIMIT_PER_BMO_IN_MB)); UInt16 numBMOsInFrag = (UInt16)generator->getFragmentDir()->getNumBMOs(); Lng32 numStreams; - double bmoMemoryUsagePerNode = getEstimatedRunTimeMemoryUsage(TRUE, &numStreams).value(); + double bmoMemoryUsagePerNode = generator->getEstMemPerNode(getKey(), numStreams); double memQuota = 0; double memQuotaRatio; if (mmu != 0) @@ -1282,22 +1282,28 @@ void PhysSequence::computeHistoryParams(Lng32 histRecLength, } -CostScalar PhysSequence::getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStreams) +CostScalar PhysSequence::getEstimatedRunTimeMemoryUsage(Generator *generator, NABoolean perNode, Lng32 *numStreams) { // input param is not used as this operator does not participate in the // quota system. ValueIdSet outputFromChild = child(0).getGroupAttr()->getCharacteristicOutputs(); + //TODO: Line below dumps core at times + //const CostScalar maxCard = child(0).getGroupAttr()->getResultMaxCardinalityForEmptyInput(); + const CostScalar maxCard = 0; + const CostScalar rowCount = numHistoryRows(); //ValueIdSet historyIds; //getHistoryAttributes(sequenceFunctions(),outputFromChild, historyIds); //historyIds += sequenceFunctions(); const Lng32 historyBufferWidthInBytes = getEstHistoryRowLength(); //historyIds.getRowLength(); - const double historyBufferSizeInBytes = numHistoryRows() * + const double historyBufferSizeInBytes = rowCount.value() * historyBufferWidthInBytes; // totalMemory is per CPU at this point of time. double totalMemory = historyBufferSizeInBytes; + CostScalar estMemPerNode; + CostScalar estMemPerInst; const PhysicalProperty* const phyProp = getPhysicalProperty(); Lng32 numOfStreams = 1; @@ -1312,19 +1318,15 @@ CostScalar PhysSequence::getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 } if (numStreams != NULL) *numStreams = numOfStreams; - if (perNode) - totalMemory /= MINOF(MAXOF(((NAClusterInfoLinux*)gpClusterInfo)->getTotalNumberOfCPUs(), 1), numOfStreams); + estMemPerNode = totalMemory /= MINOF(MAXOF(gpClusterInfo->getTotalNumberOfCPUs(), 1), numOfStreams); + estMemPerInst = totalMemory /= numOfStreams; + OperBMOQuota *operBMOQuota = new (generator->wHeap()) OperBMOQuota(getKey(), numOfStreams, + estMemPerNode, estMemPerInst, rowCount, maxCard); + generator->getBMOQuotaMap()->insert(operBMOQuota); + if (perNode) + return estMemPerNode; else - totalMemory /= numOfStreams; - return totalMemory; -} - -double PhysSequence::getEstimatedRunTimeMemoryUsage(ComTdb * tdb) -{ - Lng32 numOfStreams = 1; - CostScalar totalMemory = getEstimatedRunTimeMemoryUsage(FALSE, &numOfStreams); - totalMemory = totalMemory * numOfStreams ; - return totalMemory.value(); + return estMemPerInst; } ExplainTuple* http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/generator/Generator.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/Generator.cpp b/core/sql/generator/Generator.cpp index 3f061b8..20f799a 100644 --- a/core/sql/generator/Generator.cpp +++ b/core/sql/generator/Generator.cpp @@ -104,6 +104,7 @@ Generator::Generator(CmpContext* currentCmpContext) : ,insertNodesList_(wHeap()) ,avgVarCharSizeList_(wHeap()) ,trafSimTableInfoList_(wHeap()) + ,bmoQuotaMap_(wHeap()) { // nothing generated yet. genObj = 0; @@ -163,24 +164,11 @@ Generator::Generator(CmpContext* currentCmpContext) : tempSpace_ = NULL; numBMOs_ = 0; - totalNumBMOsPerNode_ = 0; - BMOsMemoryPerFrag_ = 0; totalBMOsMemoryPerNode_ = 0; - nBMOsMemoryPerNode_ = 0; - BMOsMemoryLimitPerNode_ = 0; - totalNumBMOsPerNode_ = 0; - - BMOsMemoryPerFrag_ = 0; - totalBMOsMemoryPerNode_ = 0; - - nBMOsMemoryPerNode_ = 0; - - BMOsMemoryLimitPerNode_ = 0; - totalNumBMOs_ = 0; numESPs_ = 1; @@ -208,7 +196,6 @@ Generator::Generator(CmpContext* currentCmpContext) : largeQueueSize_ = 0; totalEstimatedMemory_ = 0.0; - totalOverflowMemory_ = 0.0; operEstimatedMemory_ = 0; maxCpuUsage_ = 0; @@ -564,7 +551,7 @@ void Generator::genCode(const char *source, RelExpr * expr_node) disableExplain(); foundAnUpdate_ = FALSE; - + // walk through the tree of RelExpr and ItemExpr objects, generating // ComTdb, ex_expr and their relatives expr_node->codeGen(this); @@ -3386,3 +3373,90 @@ void Generator::setHBaseParallelScanner(ComTdbHbaseAccess::HbasePerfAttributes * hbpa->setDopParallelScanner(CmpCommon::getDefaultNumeric(HBASE_DOP_PARALLEL_SCANNER)); } +double Generator::getEstMemPerNode(NAString *key, Lng32 &numStreams) +{ + OperBMOQuota *operBMOQuota = bmoQuotaMap_.get(key); + if (operBMOQuota != NULL) { + numStreams = operBMOQuota->getNumStreams(); + return operBMOQuota->getEstMemPerNode(); + } else { + numStreams = 0; + return 0; + } +} + +double Generator::getEstMemForTdb(NAString *key) +{ + OperBMOQuota *operBMOQuota = bmoQuotaMap_.get(key); + if (operBMOQuota != NULL) + return operBMOQuota->getEstMemForTdb(); + else + return 0; +} + +double Generator::getEstMemPerInst(NAString *key) +{ + OperBMOQuota *operBMOQuota = bmoQuotaMap_.get(key); + if (operBMOQuota != NULL) + return operBMOQuota->getEstMemPerInst(); + else + return 0; +} + +void Generator::finetuneBMOEstimates() +{ + if (bmoQuotaMap_.entries() == 1) + return; + double bmoMemoryLimitPerNode = ActiveSchemaDB()->getDefaults().getAsDouble(BMO_MEMORY_LIMIT_PER_NODE_IN_MB); + if (bmoMemoryLimitPerNode == 0) + return; + NAHashDictionaryIterator<NAString, OperBMOQuota> iter (bmoQuotaMap_) ; + + double capMemoryRatio = ActiveSchemaDB()->getDefaults().getAsDouble(BMO_MEMORY_ESTIMATE_RATIO_CAP); + double bmoMemoryEstOutlier = + ActiveSchemaDB()->getDefaults().getAsDouble(BMO_MEMORY_ESTIMATE_OUTLIER_FACTOR) * bmoMemoryLimitPerNode * 1024 * 1024; + + double totalEstMemPerNode = totalBMOsMemoryPerNode_.value(); + double bmoMemoryRatio; + double calcTotalEstMemPerNode = 0; + double calcOperEstMemPerNode; + + NAString* key; + OperBMOQuota *operBMOQuota; + // Find the outliers and set it to the tolerable value first + iter.reset(); + iter.getNext(key,operBMOQuota); + while(key) { + calcOperEstMemPerNode = operBMOQuota->getEstMemPerNode(); + if (calcOperEstMemPerNode > bmoMemoryEstOutlier) { + operBMOQuota->setEstMemPerNode(bmoMemoryEstOutlier); + calcTotalEstMemPerNode += bmoMemoryEstOutlier; + } + else + calcTotalEstMemPerNode += calcOperEstMemPerNode; + iter.getNext(key,operBMOQuota); + } + totalBMOsMemoryPerNode_ = calcTotalEstMemPerNode; + + // Then check for the CAP to adjust it again + totalEstMemPerNode = totalBMOsMemoryPerNode_.value(); + calcTotalEstMemPerNode = 0; + iter.reset(); + iter.getNext(key,operBMOQuota); + while(key) { + calcOperEstMemPerNode = operBMOQuota->getEstMemPerNode(); + bmoMemoryRatio = calcOperEstMemPerNode / totalEstMemPerNode; + if (capMemoryRatio > 0 && capMemoryRatio <=1 && bmoMemoryRatio > capMemoryRatio) { + bmoMemoryRatio = capMemoryRatio; + calcOperEstMemPerNode = bmoMemoryRatio * calcOperEstMemPerNode; + operBMOQuota->setEstMemPerNode(calcOperEstMemPerNode); + calcTotalEstMemPerNode += calcOperEstMemPerNode; + } + else + calcTotalEstMemPerNode += calcOperEstMemPerNode; + iter.getNext(key,operBMOQuota); + } + totalBMOsMemoryPerNode_ = calcTotalEstMemPerNode; +} + + http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/generator/Generator.h ---------------------------------------------------------------------- diff --git a/core/sql/generator/Generator.h b/core/sql/generator/Generator.h index b398703..e4524a8 100644 --- a/core/sql/generator/Generator.h +++ b/core/sql/generator/Generator.h @@ -72,6 +72,7 @@ class ComTdb; class Attributes; class DP2Insert; class TrafSimilarityTableInfo; +class OperBMOQuota; // this define is used to raise assertion in generator. // Calls GeneratorAbort which does a longjmp out of the calling scope. @@ -86,6 +87,15 @@ struct CifAvgVarCharSizeCache double avgSize; }; + +class XBMOQuotaMap : public NAKeyLookup<NAString, OperBMOQuota> +{ +public: + XBMOQuotaMap(CollHeap *heap) + : NAKeyLookup<NAString, OperBMOQuota>(10, NAKeyLookupEnums::KEY_INSIDE_VALUE, heap) + {} +}; + ////////////////////////////////////////////////////////////////////////// // class Generator ////////////////////////////////////////////////////////////////////////// @@ -425,9 +435,7 @@ class Generator : public NABasicObject // temporary value holder (during pre code gen) for #BMOs in this fragment unsigned short numBMOs_; - unsigned short totalNumBMOsPerNode_; // accumulated # of BMO, per Node - CostScalar BMOsMemoryPerFrag_; // accumulated BMO memory, per fragment CostScalar totalBMOsMemoryPerNode_; // accumulated BMO memory, per Node CostScalar nBMOsMemoryPerNode_; // accumulated nBMO memory, per Node @@ -555,9 +563,6 @@ private: // total estimated memory used by BMOs and certain other operators in bytes double totalEstimatedMemory_ ; - // total overflowed memory used by Sort, HashGroupBy and HashJoin in bytes - double totalOverflowMemory_ ; - // estimated memory for an individual operator. Used by Explain // set to 0 after Explain has been called so that next operator // can used this field. In KB and on a per Node basis. @@ -628,6 +633,7 @@ private: return FALSE; }; + XBMOQuotaMap bmoQuotaMap_; public: enum cri_desc_type { UP, DOWN @@ -1460,25 +1466,13 @@ public: ItemExpr * childNode0, ItemExpr * childNode1, ComDiagsArea * diagsArea); - inline CostScalar getBMOsMemory() { return BMOsMemoryPerFrag_; } - inline void incrBMOsMemory(CostScalar x) - { incrBMOsMemoryPerFrag(x); totalBMOsMemoryPerNode_ += x; } - - inline void incrBMOsMemoryPerFrag(CostScalar x) - { BMOsMemoryPerFrag_ += x; } + { totalBMOsMemoryPerNode_ += x; } - inline CostScalar replaceBMOsMemoryUsage(CostScalar newVal) - { - CostScalar retVal = BMOsMemoryPerFrag_; - BMOsMemoryPerFrag_ = newVal; - return retVal; - } inline CostScalar getTotalBMOsMemoryPerNode() - { return totalBMOsMemoryPerNode_; } - + { return totalBMOsMemoryPerNode_; } inline void incrNumBMOs() - { incrNumBMOsPerFrag(1); totalNumBMOsPerNode_++; totalNumBMOs_++;} + { incrNumBMOsPerFrag(1); totalNumBMOs_++;} inline void incrNumBMOsPerFrag(UInt32 x) { numBMOs_ += x; } @@ -1488,11 +1482,10 @@ public: numBMOs_ = newVal; return retVal; } - inline unsigned short getTotalNumBMOsPerNode() { return totalNumBMOsPerNode_; } - + inline CostScalar getTotalNBMOsMemoryPerNode() { return nBMOsMemoryPerNode_; } inline void incrNBMOsMemoryPerNode(CostScalar x) { nBMOsMemoryPerNode_ += x; } - + inline void setBMOsMemoryLimitPerNode(CostScalar x) { BMOsMemoryLimitPerNode_ = x; } @@ -1605,10 +1598,6 @@ public: inline short getMaxCpuUsage(){return maxCpuUsage_;} inline void setMaxCpuUsage(short val){maxCpuUsage_ = val;} - inline double getTotalOverflowMemory(){return totalOverflowMemory_;} - inline void addToTotalOverflowMemory(double val) - {totalOverflowMemory_ += val;} - inline ComTdb::OverflowModeType getOverflowMode() {return overflowMode_; } // Each of these two mutators return the old value @@ -1692,7 +1681,11 @@ public: inline void setTopNRows(ULng32 topNRows) { topNRows_ = topNRows; } inline ULng32 getTopNRows() { return topNRows_; } - + inline XBMOQuotaMap *getBMOQuotaMap() { return &bmoQuotaMap_; } + double getEstMemPerNode(NAString *key, Lng32 &numStreams); + double getEstMemForTdb(NAString *key); + double getEstMemPerInst(NAString *key); + void finetuneBMOEstimates(); }; // class Generator class GenOperSimilarityInfo : public NABasicObject @@ -1731,6 +1724,46 @@ private: UInt32 flags_; }; + +class OperBMOQuota : public NABasicObject +{ +public: + OperBMOQuota(NAString *operAddr, Int32 numStreams, CostScalar estMemPerNode, CostScalar estMemPerInst, + CostScalar estRowsUsed, CostScalar maxCard) : + operAddr_(operAddr) + , numStreams_(numStreams) + , estMemPerNode_(estMemPerNode) + , estMemPerInst_(estMemPerInst) + , estRowsUsed_(estRowsUsed) + , maxCard_(maxCard) + , ignoreEstimate_(FALSE) + , origEstMemPerNode_(estMemPerNode) + { + //weight_ = (estRowsUsed_ / maxCard_).value(); + weight_ = 0; + } + const NAString *getKey() const {return operAddr_; } + inline Int32 getNumStreams() { return numStreams_; } + inline double getEstMemPerNode() { return estMemPerNode_.value(); } + inline double getEstMemPerInst() { return estMemPerInst_.value(); } + inline double getEstMemForTdb() { return estMemPerInst_.value() * numStreams_; } + inline void setIgnoreEstimate() { ignoreEstimate_ = TRUE; } + NABoolean operator==(const OperBMOQuota &other) const + { return this == &other; } + inline void setEstMemPerNode(double estMemPerNode) { estMemPerNode_ = estMemPerNode; } + +private: + const NAString *operAddr_; + Int32 numStreams_; + CostScalar estMemPerNode_; + CostScalar estMemPerInst_; + CostScalar estRowsUsed_; + CostScalar maxCard_; + CostScalar origEstMemPerNode_; + double weight_; + NABoolean ignoreEstimate_; +}; + // Get table and index filename extern const NAString GenGetQualifiedName(const CorrName&, NABoolean formatForDisplay = FALSE); @@ -1784,5 +1817,6 @@ Generator::getExplainTuple() return explainTuple_; } + #endif http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/optimizer/RelEnforcer.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelEnforcer.h b/core/sql/optimizer/RelEnforcer.h index e6ebc12..752d1ec 100644 --- a/core/sql/optimizer/RelEnforcer.h +++ b/core/sql/optimizer/RelEnforcer.h @@ -188,10 +188,7 @@ public: virtual NABoolean isBigMemoryOperator(const PlanWorkSpace* pws, const Lng32 planNumber); - virtual CostScalar getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStreams = NULL); - virtual double getEstimatedRunTimeMemoryUsage(ComTdb * tdb); - - virtual double getEstimatedRunTimeOverflowSize(double memoryQuotaMB); + virtual CostScalar getEstimatedRunTimeMemoryUsage(Generator *generator, NABoolean perNode, Lng32 *numStreams = NULL); virtual PlanPriority computeOperatorPriority (const Context* context, @@ -585,8 +582,8 @@ public: inline void setBMOsMemoryUsage(CostScalar x) { BMOsMemoryUsage_ = x; } inline CostScalar getBMOsMemoryUsage() { return BMOsMemoryUsage_ ; } - virtual CostScalar getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStreams = NULL); - virtual double getEstimatedRunTimeMemoryUsage(ComTdb * tdb); + virtual CostScalar getEstimatedRunTimeMemoryUsage(Generator *generator, NABoolean perNode, Lng32 *numStreams = NULL); + virtual double getEstimatedRunTimeMemoryUsage(Generator *generator, ComTdb * tdb); void setExtractProducerFlag() { isExtractProducer_ = TRUE; } NABoolean getExtractProducerFlag() { return isExtractProducer_; } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/optimizer/RelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExpr.cpp b/core/sql/optimizer/RelExpr.cpp index 4f2f295..3d72f0d 100644 --- a/core/sql/optimizer/RelExpr.cpp +++ b/core/sql/optimizer/RelExpr.cpp @@ -288,6 +288,7 @@ RelExpr::RelExpr(OperatorTypeEnum otype, ,cachedResizeCIFRecord_(FALSE) ,dopReduced_(FALSE) ,originalExpr_(NULL) + ,operKey_(outHeap) { child_[0] = leftChild; @@ -2042,12 +2043,6 @@ double RelExpr::computeMemoryQuota(NABoolean inMaster, variableMemLimit = (1-equalQuotaShareRatio) * exeMem; } double bmoMemoryRatio = bmoMemoryUsage / totalBMOsMemoryUsage; - double capMemoryRatio = 1; - if (totalNumBMOs > 1) { - capMemoryRatio = ActiveSchemaDB()->getDefaults().getAsDouble(BMO_MEMORY_ESTIMATE_RATIO_CAP); - if (capMemoryRatio > 0 && capMemoryRatio <=1 && bmoMemoryRatio > capMemoryRatio) - bmoMemoryRatio = capMemoryRatio; - } bmoQuotaRatio = bmoMemoryRatio; double bmoMemoryQuotaPerNode = constMemQuota + (variableMemLimit * bmoMemoryRatio); double numInstancesPerNode = numStreams / MINOF(MAXOF(((NAClusterInfoLinux*)gpClusterInfo)->getTotalNumberOfCPUs(), 1), numStreams); @@ -16309,3 +16304,14 @@ CostScalar RelExpr::getChild0Cardinality(const Context* context) return ch0RowCount; } +NAString *RelExpr::getKey() +{ + + if (operKey_.length() == 0) + { + char keyBuffer[30]; + snprintf(keyBuffer, sizeof(keyBuffer), "%ld", (Int64)this); + operKey_ = keyBuffer; + } + return &operKey_; +} http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/optimizer/RelExpr.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelExpr.h b/core/sql/optimizer/RelExpr.h index 90be900..3074aa8 100644 --- a/core/sql/optimizer/RelExpr.h +++ b/core/sql/optimizer/RelExpr.h @@ -1291,13 +1291,9 @@ public: // --------------------------------------------------------------------- virtual NABoolean isBigMemoryOperator(const PlanWorkSpace* pws, const Lng32 planNumber); -/* - virtual CostScalar getEstimatedRunTimeMemoryUsageInMB(NABoolean perNode) - { return getEstimatedRunTimeMemoryUsage(perNode) / (1024*1024); } -*/ - virtual CostScalar getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStreams = NULL) {return 0;} - virtual double getEstimatedRunTimeMemoryUsage(ComTdb * tdb) {return 0;} + virtual CostScalar getEstimatedRunTimeMemoryUsage(Generator *generator, NABoolean perNode, Lng32 *numStreams = NULL) {return 0;} + virtual double getEstimatedRunTimeMemoryUsage(Generator *generator, ComTdb * tdb) {return 0;} inline NABoolean isinBlockStmt() const { return isinBlockStmt_; } @@ -1598,6 +1594,8 @@ private: // remember the original here, e.g. to find VEG regions RelExpr *originalExpr_; + NAString operKey_; + public: // begin: accessors & mutators for relexpr tracking info @@ -1640,7 +1638,7 @@ public: CostScalar getChild0Cardinality(const Context*); - + NAString *getKey(); }; // class RelExpr // ----------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/optimizer/RelFastTransport.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelFastTransport.cpp b/core/sql/optimizer/RelFastTransport.cpp index 8773b5c..9981b6c 100644 --- a/core/sql/optimizer/RelFastTransport.cpp +++ b/core/sql/optimizer/RelFastTransport.cpp @@ -457,7 +457,7 @@ PhysicalProperty* PhysicalFastExtract::synthPhysicalProperty(const Context* myCo return sppForMe ; }; -double PhysicalFastExtract::getEstimatedRunTimeMemoryUsage(ComTdb * tdb) +double PhysicalFastExtract::getEstimatedRunTimeMemoryUsage(Generator *generator, ComTdb * tdb) { // The executor attempts to get buffers, each of size 1 MB. This memory http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/optimizer/RelFastTransport.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelFastTransport.h b/core/sql/optimizer/RelFastTransport.h index 4a4acb7..8edfb61 100644 --- a/core/sql/optimizer/RelFastTransport.h +++ b/core/sql/optimizer/RelFastTransport.h @@ -408,7 +408,7 @@ public: const ValueIdSet & externalInputs, ValueIdSet &pulledNewInputs); - virtual double getEstimatedRunTimeMemoryUsage(ComTdb * tdb) ; + virtual double getEstimatedRunTimeMemoryUsage(Generator *generator, ComTdb * tdb) ; virtual short codeGen(Generator *); static NABoolean isSpecialChar(char * str , char & chr); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/optimizer/RelGrby.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelGrby.h b/core/sql/optimizer/RelGrby.h index 28d0fbc..fc3e486 100644 --- a/core/sql/optimizer/RelGrby.h +++ b/core/sql/optimizer/RelGrby.h @@ -821,11 +821,8 @@ public: virtual NABoolean isBigMemoryOperator(const PlanWorkSpace* pws, const Lng32 planNumber); - virtual CostScalar getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStreams = NULL); - virtual double getEstimatedRunTimeMemoryUsage(ComTdb * tdb); + virtual CostScalar getEstimatedRunTimeMemoryUsage(Generator *generator, NABoolean perNode, Lng32 *numStreams = NULL); - virtual double getEstimatedRunTimeOverflowSize(double memoryQuotaMB); - virtual PlanPriority computeOperatorPriority (const Context* context, PlanWorkSpace *pws=NULL, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/optimizer/RelJoin.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelJoin.h b/core/sql/optimizer/RelJoin.h index 3643210..f2eda71 100644 --- a/core/sql/optimizer/RelJoin.h +++ b/core/sql/optimizer/RelJoin.h @@ -1631,10 +1631,7 @@ public: virtual NABoolean isBigMemoryOperator(const PlanWorkSpace* pws, const Lng32 planNumber); - virtual CostScalar getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStreams = NULL); - virtual double getEstimatedRunTimeMemoryUsage(ComTdb * tdb); - - virtual double getEstimatedRunTimeOverflowSize(double memoryQuotaMB); + virtual CostScalar getEstimatedRunTimeMemoryUsage(Generator *generator, NABoolean perNode, Lng32 *numStreams = NULL); inline ValueIdSet & checkInputValues() { return checkInputValues_;} inline ValueIdSet & moveInputValues() { return moveInputValues_;} http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/optimizer/RelProbeCache.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelProbeCache.h b/core/sql/optimizer/RelProbeCache.h index bc79a3d..ac2f0f7 100644 --- a/core/sql/optimizer/RelProbeCache.h +++ b/core/sql/optimizer/RelProbeCache.h @@ -74,8 +74,8 @@ public: virtual short codeGen(Generator *g); - virtual CostScalar getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStreams = NULL); - virtual double getEstimatedRunTimeMemoryUsage(ComTdb * tdb); + virtual CostScalar getEstimatedRunTimeMemoryUsage(Generator *generator, NABoolean perNode, Lng32 *numStreams = NULL); + virtual double getEstimatedRunTimeMemoryUsage(Generator *generator, ComTdb * tdb); virtual const NAString getText() const; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/optimizer/RelRoutine.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelRoutine.cpp b/core/sql/optimizer/RelRoutine.cpp index 336c3ab..20c780c 100644 --- a/core/sql/optimizer/RelRoutine.cpp +++ b/core/sql/optimizer/RelRoutine.cpp @@ -1186,7 +1186,7 @@ NARoutine * PredefinedTableMappingFunction::getRoutineMetadata( // ----------------------------------------------------------------------- // methods for class PhysicalTableMappingUDF // ----------------------------------------------------------------------- -double PhysicalTableMappingUDF::getEstimatedRunTimeMemoryUsage(ComTdb * tdb) {return 0;} +double PhysicalTableMappingUDF::getEstimatedRunTimeMemoryUsage(Generator *generator, ComTdb * tdb) {return 0;} RelExpr * PhysicalTableMappingUDF::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap) http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/optimizer/RelRoutine.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelRoutine.h b/core/sql/optimizer/RelRoutine.h index ca63321..05a74e1 100644 --- a/core/sql/optimizer/RelRoutine.h +++ b/core/sql/optimizer/RelRoutine.h @@ -1105,7 +1105,7 @@ public: const ValueIdSet & externalInputs, ValueIdSet &pulledNewInputs); - virtual double getEstimatedRunTimeMemoryUsage(ComTdb * tdb) ; + virtual double getEstimatedRunTimeMemoryUsage(Generator *generator, ComTdb * tdb) ; virtual short codeGen(Generator *); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/optimizer/RelSequence.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/RelSequence.h b/core/sql/optimizer/RelSequence.h index b5069d1..189fdbe 100644 --- a/core/sql/optimizer/RelSequence.h +++ b/core/sql/optimizer/RelSequence.h @@ -430,9 +430,7 @@ public: virtual NABoolean isBigMemoryOperator(const Context* context, const Lng32 planNumber); - virtual CostScalar getEstimatedRunTimeMemoryUsage(NABoolean perNode, Lng32 *numStreams = NULL); - virtual double getEstimatedRunTimeMemoryUsage(ComTdb * tdb); - + virtual CostScalar getEstimatedRunTimeMemoryUsage(Generator *generator, NABoolean perNode, Lng32 *numStreams = NULL); // Redefine these virtual methods to declare this node as a // physical node. http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/regress/hive/EXPECTED030 ---------------------------------------------------------------------- diff --git a/core/sql/regress/hive/EXPECTED030 b/core/sql/regress/hive/EXPECTED030 index 7539f28..4ec4628 100644 --- a/core/sql/regress/hive/EXPECTED030 +++ b/core/sql/regress/hive/EXPECTED030 @@ -795,7 +795,6 @@ DESCRIPTION statement_index ........ 0 affinity_value ......... 0 max_max_cardinal 1,440,202 - total_overflow_size .... 0.00 KB esp_2_node_map ......... (\NSK:-1:-1:-1:-1) xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -912,7 +911,6 @@ DESCRIPTION statement_index ........ 0 affinity_value ......... 0 max_max_cardinality .... 1 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -1274,7 +1272,6 @@ DESCRIPTION statement_index ........ 0 affinity_value ......... 0 max_max_cardinality 1,698 - total_overflow_size .... 0.00 KB esp_2_node_map ......... (\NSK:-1:-1:-1:-1) xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -1418,7 +1415,6 @@ DESCRIPTION statement_index ........ 0 affinity_value ......... 0 max_max_cardinal 2,880,404 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -1808,7 +1804,6 @@ DESCRIPTION statement_index ........ 0 affinity_value ......... 0 max_max_cardinality 271 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -2057,7 +2052,6 @@ DESCRIPTION statement_index ........ 0 affinity_value ......... 0 max_max_cardinality 271 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -2352,7 +2346,6 @@ DESCRIPTION statement_index ........ 0 affinity_value ......... 0 max_max_cardinal 2,880,404 - total_overflow_size .... 0.00 KB esp_2_node_map ......... (\NSK:-1:-1:-1:-1) esp_3_node_map ......... (\NSK:-1:-1) xn_access_mode ......... read_only http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/regress/seabase/EXPECTED010 ---------------------------------------------------------------------- diff --git a/core/sql/regress/seabase/EXPECTED010 b/core/sql/regress/seabase/EXPECTED010 index d0662c3..314ad27 100644 --- a/core/sql/regress/seabase/EXPECTED010 +++ b/core/sql/regress/seabase/EXPECTED010 @@ -241,7 +241,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality ... 11 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -325,7 +324,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 1 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -407,7 +405,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality ... 10 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -491,7 +488,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 1 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -838,7 +834,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -918,7 +913,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 5 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -998,7 +992,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -1077,7 +1070,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -1186,7 +1178,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -1259,7 +1250,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -1340,7 +1330,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 5 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -1421,7 +1410,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -1501,7 +1489,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -1611,7 +1598,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -1995,7 +1981,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality ... 11 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -2079,7 +2064,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 1 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -2160,7 +2144,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality ... 10 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -2243,7 +2226,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 1 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -2550,7 +2532,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -2630,7 +2611,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 5 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -2722,7 +2702,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -2804,7 +2783,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -2918,7 +2896,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -2993,7 +2970,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -3075,7 +3051,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 5 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -3157,7 +3132,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -3238,7 +3212,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -3350,7 +3323,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -3734,7 +3706,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality ... 11 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -3818,7 +3789,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 1 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -3900,7 +3870,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality ... 10 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -3984,7 +3953,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 1 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -4292,7 +4260,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -4373,7 +4340,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 5 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -4454,7 +4420,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -4534,7 +4499,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -4645,7 +4609,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -4717,7 +4680,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -4799,7 +4761,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 5 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -4881,7 +4842,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -4962,7 +4922,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -5074,7 +5033,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -5458,7 +5416,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality ... 11 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -5542,7 +5499,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 1 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -5623,7 +5579,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality ... 10 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -5706,7 +5661,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 1 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -6052,7 +6006,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -6131,7 +6084,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 5 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -6210,7 +6162,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -6288,7 +6239,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -6396,7 +6346,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -6472,7 +6421,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -6552,7 +6500,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 5 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -6632,7 +6579,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -6711,7 +6657,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -6820,7 +6765,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -7227,7 +7171,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality ... 11 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -7311,7 +7254,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 1 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -7393,7 +7335,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality ... 10 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -7477,7 +7418,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 1 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -7824,7 +7764,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -7904,7 +7843,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 5 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -7984,7 +7922,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -8063,7 +8000,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -8172,7 +8108,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -8245,7 +8180,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -8326,7 +8260,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 5 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -8407,7 +8340,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -8487,7 +8419,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -8597,7 +8528,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -8981,7 +8911,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality ... 11 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -9065,7 +8994,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 1 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -9146,7 +9074,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality ... 10 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -9229,7 +9156,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 1 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -9575,7 +9501,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -9654,7 +9579,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 5 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -9733,7 +9657,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -9811,7 +9734,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -9919,7 +9841,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -9995,7 +9916,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -10075,7 +9995,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 5 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -10155,7 +10074,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled @@ -10234,7 +10152,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 4 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -10343,7 +10260,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality .... 2 - total_overflow_size .... 0.00 KB upd_action_on_error .... xn_rollback xn_access_mode ......... read_only xn_autoabort_interval 0 http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/regress/seabase/EXPECTED011 ---------------------------------------------------------------------- diff --git a/core/sql/regress/seabase/EXPECTED011 b/core/sql/regress/seabase/EXPECTED011 index 2a36907..e186f9c 100644 --- a/core/sql/regress/seabase/EXPECTED011 +++ b/core/sql/regress/seabase/EXPECTED011 @@ -163,7 +163,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinality ... 11 - total_overflow_size .... 0.00 KB xn_access_mode ......... read_only xn_autoabort_interval 0 auto_query_retry ....... enabled http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/regress/seabase/EXPECTED016 ---------------------------------------------------------------------- diff --git a/core/sql/regress/seabase/EXPECTED016 b/core/sql/regress/seabase/EXPECTED016 index 4be8bfc..05ce633 100644 --- a/core/sql/regress/seabase/EXPECTED016 +++ b/core/sql/regress/seabase/EXPECTED016 @@ -201,7 +201,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinal 1,000,000,000 - total_overflow_size .... 0.00 KB esp_2_node_map ......... (\NSK:-1:-1:-1:-1) xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -349,7 +348,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinal 1,000,000,000 - total_overflow_size .... 0.00 KB esp_2_node_map ......... (\NSK:-1:-1:-1:-1) xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -497,7 +495,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinal 1,000,000,000 - total_overflow_size .... 0.00 KB esp_2_node_map ......... (\NSK:-1:-1:-1:-1) xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -645,7 +642,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinal 1,000,000,000 - total_overflow_size .... 0.00 KB esp_2_node_map ......... (\NSK:-1:-1:-1:-1) xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -793,7 +789,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinal 1,000,000,000 - total_overflow_size .... 0.00 KB esp_2_node_map ......... (\NSK:-1:-1:-1:-1) xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -941,7 +936,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinal 1,000,000,000 - total_overflow_size .... 0.00 KB esp_2_node_map ......... (\NSK:-1:-1:-1:-1) xn_access_mode ......... read_only xn_autoabort_interval 0 @@ -1089,7 +1083,6 @@ DESCRIPTION fragment_type .......... master affinity_value ......... 0 max_max_cardinal 1,000,000,000 - total_overflow_size .... 0.00 KB esp_2_node_map ......... (\NSK:-1:-1:-1:-1) xn_access_mode ......... read_only xn_autoabort_interval 0 http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/sqlcomp/DefaultConstants.h ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h index 9fb5d22..d84455b 100644 --- a/core/sql/sqlcomp/DefaultConstants.h +++ b/core/sql/sqlcomp/DefaultConstants.h @@ -3895,7 +3895,8 @@ enum DefaultConstants MEMORY_LIMIT_ROWSET_IN_MB, SUPPRESS_CHAR_LIMIT_CHECK, - + + BMO_MEMORY_ESTIMATE_OUTLIER_FACTOR, // This enum constant must be the LAST one in the list; it's a count, // not an Attribute (it's not IN DefaultDefaults; it's the SIZE of it)! __NUM_DEFAULT_ATTRIBUTES http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/375eeeab/core/sql/sqlcomp/nadefaults.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp index d3448ee..0cc0fba 100644 --- a/core/sql/sqlcomp/nadefaults.cpp +++ b/core/sql/sqlcomp/nadefaults.cpp @@ -462,6 +462,7 @@ SDDkwd__(ALLOW_DP2_ROW_SAMPLING, "SYSTEM"), DDflte_(BMO_CITIZENSHIP_FACTOR, "1."), DDflte_(BMO_MEMORY_EQUAL_QUOTA_SHARE_RATIO, "0.5"), + DDui___(BMO_MEMORY_ESTIMATE_OUTLIER_FACTOR, "10"), DDflte_(BMO_MEMORY_ESTIMATE_RATIO_CAP, "0.7"), DDui___(BMO_MEMORY_LIMIT_LOWER_BOUND_HASHGROUPBY , "25"), DDui___(BMO_MEMORY_LIMIT_LOWER_BOUND_HASHJOIN, "25"),
