Github user selvaganesang commented on a diff in the pull request: https://github.com/apache/incubator-trafodion/pull/1228#discussion_r138498956 --- Diff: core/sql/optimizer/RelExpr.cpp --- @@ -1988,77 +1988,62 @@ NABoolean RelExpr::containsNode(OperatorTypeEnum nodeType) } double RelExpr::computeMemoryQuota(NABoolean inMaster, - NABoolean perCPU, - double BMOsMemoryLimit, // in bytes - UInt16 totalNumBMOs, // per CPU - double totalBMOsMemoryUsage, // per CPU, in bytes + NABoolean perNode, + double BMOsMemoryLimit, // in MB + UInt16 totalNumBMOs, // per query + double totalBMOsMemoryUsage, // for all BMOs per node in bytes UInt16 numBMOsPerFragment, // per fragment - double BMOsMemoryUsagePerFragment // per fragment, in bytes + double bmoMemoryUsage, // for the current BMO/Operator per node in bytes + Lng32 numStreams, + double &bmoQuotaRatio ) { - if ( perCPU == TRUE ) { - Lng32 exeMem = Lng32(BMOsMemoryLimit/(1024*1024)); + if ( perNode == TRUE ) { + Lng32 exeMem = Lng32(BMOsMemoryLimit/(1024*1024)); - if ( inMaster && CmpCommon::getDefault(ODBC_PROCESS) == DF_ON ) { - - // Limiting the total memory in the master process when in both - // the per-CPU estimation and the ODBC mode. - - NADefaults &defs = ActiveSchemaDB()->getDefaults(); - - Lng32 inCpuLimitDelta = - defs.getAsLong(EXE_MEMORY_AVAILABLE_IN_MB) - - - defs.getAsLong(EXE_MEMORY_RESERVED_FOR_MXOSRVR_IN_MB); - - if ( inCpuLimitDelta < 0 ) - inCpuLimitDelta = 50; - - if (exeMem > inCpuLimitDelta) - exeMem = inCpuLimitDelta; + // the quota is allocated in 2 parts + // The constant part divided equally across all bmo operators + // The variable part allocated in proportion of the given BMO operator + // estimated memory usage to the total estimated memory usage of all BMOs + + // The ratio can be capped by the CQD + double equalQuotaShareRatio = 0; + equalQuotaShareRatio = ActiveSchemaDB()->getDefaults().getAsDouble(BMO_MEMORY_EQUAL_QUOTA_SHARE_RATIO); +/* + char *equalQuotaShareRatioStr = getenv("BMO_MEMORY_EQUAL_QUOTA_SHARE_RATIO"); + if (equalQuotaShareRatioStr != NULL) + equalQuotaShareRatio = atof(equalQuotaShareRatioStr); +*/ + double constMemQuota = 0; + double variableMemLimit = exeMem; + if (equalQuotaShareRatio > 0 && totalNumBMOs > 1) { + constMemQuota = (exeMem * equalQuotaShareRatio )/ totalNumBMOs; + variableMemLimit = (1-equalQuotaShareRatio) * exeMem; } - - // the quota is propotional to both the # of BMOs and the estimated memory - // usage in the fragment, and evenly distrbuted among BMOs in the fragment. - return ((exeMem/2) * (BMOsMemoryUsagePerFragment/totalBMOsMemoryUsage + - double(numBMOsPerFragment)/totalNumBMOs) - ) / numBMOsPerFragment; + double bmoMemoryRatio = bmoMemoryUsage / totalBMOsMemoryUsage; + double capMemoryRatio = 1; + if (totalNumBMOs > 1) { + capMemoryRatio = ActiveSchemaDB()->getDefaults().getAsDouble(BMO_MEMORY_ESTIMATE_RATIO_CAP); + if (capMemoryRatio > 0 && capMemoryRatio <=1 && bmoMemoryRatio > capMemoryRatio) + bmoMemoryRatio = capMemoryRatio; --- End diff -- Yes. That's true. In that case, we need to do multiple pass or adjust the totalBMOMemoryUsage for the unassigned BMO operator. Will attempt to do this later
---