HAWQ-563. Restore guc hawq_rm_nvseg_variance_amon_seg_respool_limit and fix its bug when resource manager resets resource
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/802d537c Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/802d537c Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/802d537c Branch: refs/heads/HAWQ-459 Commit: 802d537cc8e19618fd2553608746ae595d8f8ec2 Parents: 0bb22dc Author: YI JIN <[email protected]> Authored: Mon Mar 21 15:47:04 2016 +1100 Committer: YI JIN <[email protected]> Committed: Mon Mar 21 15:47:04 2016 +1100 ---------------------------------------------------------------------- src/backend/cdb/cdbvars.c | 2 ++ .../resourcemanager/include/resourcepool.h | 1 + src/backend/resourcemanager/resourcemanager.c | 1 + src/backend/resourcemanager/resourcepool.c | 18 ++++++++++++++++-- src/backend/utils/misc/guc.c | 11 +++++++++++ 5 files changed, 31 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/cdb/cdbvars.c ---------------------------------------------------------------------- diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c index a2476fd..b4a9f3d 100644 --- a/src/backend/cdb/cdbvars.c +++ b/src/backend/cdb/cdbvars.c @@ -390,6 +390,8 @@ double rm_regularize_io_factor; double rm_regularize_usage_factor; double rm_regularize_nvseg_factor; +int rm_nvseg_variance_among_seg_respool_limit; + /* Greenplum Database Experimental Feature GUCs */ int gp_distinct_grouping_sets_threshold = 32; bool gp_enable_explain_allstat = FALSE; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/resourcemanager/include/resourcepool.h ---------------------------------------------------------------------- diff --git a/src/backend/resourcemanager/include/resourcepool.h b/src/backend/resourcemanager/include/resourcepool.h index f965c43..08ce305 100644 --- a/src/backend/resourcemanager/include/resourcepool.h +++ b/src/backend/resourcemanager/include/resourcepool.h @@ -633,6 +633,7 @@ int getOrderedResourceAllocTreeIndexByRatio(uint32_t ratio, BBST *tree); void setAllSegResourceGRMUnhandled(void); void resetAllSegmentsGRMContainerFailAllocCount(void); +void resetAllSegmentsNVSeg(void); struct RB_GRMContainerStatData { http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/resourcemanager/resourcemanager.c ---------------------------------------------------------------------- diff --git a/src/backend/resourcemanager/resourcemanager.c b/src/backend/resourcemanager/resourcemanager.c index c585d40..c7215be 100644 --- a/src/backend/resourcemanager/resourcemanager.c +++ b/src/backend/resourcemanager/resourcemanager.c @@ -580,6 +580,7 @@ int MainHandlerLoop(void) refreshMemoryCoreRatioLevelUsage(gettime_microsec()); resetAllSegmentsGRMContainerFailAllocCount(); + resetAllSegmentsNVSeg(); /* Check if can resume using new available global resource manager.*/ if ( cleanedAllGRMContainers() ) http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/resourcemanager/resourcepool.c ---------------------------------------------------------------------- diff --git a/src/backend/resourcemanager/resourcepool.c b/src/backend/resourcemanager/resourcepool.c index acc8dbe..3a8d72b 100644 --- a/src/backend/resourcemanager/resourcepool.c +++ b/src/backend/resourcemanager/resourcepool.c @@ -1650,6 +1650,20 @@ void resetAllSegmentsGRMContainerFailAllocCount(void) freePAIRRefList(&(PRESPOOL->Segments), &allsegres); } +void resetAllSegmentsNVSeg(void) +{ + List *allsegres = NULL; + ListCell *cell = NULL; + getAllPAIRRefIntoList(&(PRESPOOL->Segments), &allsegres); + + foreach(cell, allsegres) + { + SegResource segres = (SegResource)(((PAIR)lfirst(cell))->Value); + segres->NVSeg = 0; + } + freePAIRRefList(&(PRESPOOL->Segments), &allsegres); +} + /* * Check index to get host id based on host name string. */ @@ -2556,7 +2570,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t nodecount, freePAIRRefList(&(PRESPOOL->Segments), &ressegl); Assert(minnvseg <= maxnvseg); - if ( maxnvseg - minnvseg > rm_nvseg_variance_among_seg_limit ) + if ( maxnvseg - minnvseg > rm_nvseg_variance_among_seg_respool_limit ) { elog(LOG, "Reject virtual segment allocation based on data " "locality information. After tentative allocation " @@ -2565,7 +2579,7 @@ int allocateResourceFromResourcePoolIOBytes2(int32_t nodecount, "is %d, tolerated difference limit is %d.", maxnvseg, minnvseg, - rm_nvseg_variance_among_seg_limit); + rm_nvseg_variance_among_seg_respool_limit); /* Return the allocated resource. */ List *vsegcntlist = NULL; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/802d537c/src/backend/utils/misc/guc.c ---------------------------------------------------------------------- diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 56909c9..a881b31 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -6557,6 +6557,17 @@ static struct config_int ConfigureNamesInt[] = }, { + {"hawq_rm_nvseg_variance_amon_seg_respool_limit", PGC_POSTMASTER, RESOURCES_MGM, + gettext_noop("the variance of vseg number in each segment that resource manager " + "should tolerate at most in resource pool when choosing segments " + "based on data locality reference."), + NULL + }, + &rm_nvseg_variance_among_seg_respool_limit, + 2, 0, 65535, NULL, NULL + }, + + { {"hawq_rm_container_batch_limit", PGC_POSTMASTER, RESOURCES_MGM, gettext_noop("the batch process limit for global resource manager containers."), NULL
