Github user jiny2 commented on a diff in the pull request:
https://github.com/apache/incubator-hawq/pull/83#discussion_r44236932
--- Diff: src/backend/resourcemanager/resourcemanager.c ---
@@ -2177,6 +2213,149 @@ int generateAllocRequestToBroker(void)
return res;
}
+void completeAllocRequestToBroker(int32_t *reqmem,
+ int32_t
*reqcore,
+ List
**preferred)
+{
+ /*
+ * Go through each segment to get minimum water level. The idea of
completing
+ * the request is to keep pulling up the lowest water level in the
cluster
+ * until equal or more GRM containers are requested.
+ */
+ Assert(*reqmem % *reqcore == 0);
+ uint32_t ratio = *reqmem / *reqcore;
+
+ /* Step 1. Get lowest water level and build up index. */
+ List *ressegl = NULL;
+ getAllPAIRRefIntoList(&(PRESPOOL->Segments), &ressegl);
+ /* Index of each segment in current preferred host list. */
+ PAIR *reqidx = rm_palloc0(PCONTEXT, sizeof(PAIR) *
list_length(ressegl));
+ int llevel = INT_MAX;
+ int totalcount = 0;
+ int index = 0;
+ ListCell *cell = NULL;
+ foreach(cell, ressegl)
+ {
+ reqidx[index] = NULL;
+
+ PAIR pair = (PAIR)lfirst(cell);
+ SegResource segres = (SegResource)(pair->Value);
+
+ /*
+ * Resource manager skips this segment if
+ * 1) Not FTS available;
+ * 2) Not GRM available;
+ * 3) Having resource decrease pending.
+ */
+ if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
+ (DRMGlobalInstance->ImpType != NONE_HAWQ2 &&
+ !IS_SEGSTAT_GRMAVAILABLE(segres->Stat)) ||
+ (segres->DecPending.MemoryMB > 0 &&
segres->DecPending.Core > 0))
+ {
+ index++;
+ continue;
+ }
+
+ int clevel = segres->ContainerSets[0] == NULL ?
+ 0 :
+
list_length(segres->ContainerSets[0]->Containers) +
+ segres->IncPending.MemoryMB / ratio;
+
+ ListCell *pcell = NULL;
+ foreach(pcell, *preferred)
+ {
+ PAIR existpair = (PAIR)lfirst(pcell);
+ if ( pair->Value == segres )
+ {
+ reqidx[index] = existpair;
+ totalcount +=
((ResourceBundle)(reqidx[index]->Value))->MemoryMB /
+ ratio;
+ break;
+ }
+ }
+
+
+ int creqsize = reqidx[index] == NULL ?
+ 0 :
+
((ResourceBundle)(reqidx[index]->Value))->MemoryMB / ratio;
+
+ llevel = clevel+creqsize < llevel ? clevel+creqsize : llevel;
+ index++;
+ }
+
+ /* Step 2. Adjust request. */
+ int32_t reqcoreleft = *reqcore - totalcount;
+ while( reqcoreleft > 0 )
+ {
+ llevel++;
+ index = 0;
+ foreach(cell, ressegl)
+ {
+ PAIR pair = (PAIR)lfirst(cell);
+ SegResource segres = (SegResource)(pair->Value);
+
+ /*
+ * Resource manager skips this segment if
+ * 1) Not FTS available;
+ * 2) Not GRM available;
+ * 3) Having resource decrease pending.
+ */
+ if (!IS_SEGSTAT_FTSAVAILABLE(segres->Stat) ||
+ (DRMGlobalInstance->ImpType != NONE_HAWQ2 &&
+ !IS_SEGSTAT_GRMAVAILABLE(segres->Stat)) ||
+ (segres->DecPending.MemoryMB > 0 &&
segres->DecPending.Core > 0))
+ {
+ index++;
+ continue;
+ }
+
+ int clevel = segres->ContainerSets[0] == NULL ?
+ 0 :
+
list_length(segres->ContainerSets[0]->Containers) +
+ segres->IncPending.MemoryMB /
ratio;
+
+ int aclevel = reqidx[index] == NULL ?
+ clevel :
+ clevel +
((ResourceBundle)(reqidx[index]->Value))->MemoryMB / ratio;
+
+ if ( llevel > aclevel )
+ {
+
+ if ( reqidx[index] == NULL )
+ {
+ reqidx[index] = rm_palloc0(PCONTEXT,
sizeof(PAIRData));
+ reqidx[index]->Key = segres;
+ ResourceBundle resource =
rm_palloc0(PCONTEXT,
+
sizeof(ResourceBundleData));
+ resetResourceBundleData(resource, 0, 0,
ratio);
+ reqidx[index]->Value = resource;
+ *preferred = lappend(*preferred,
reqidx[index]);
+ }
+
addResourceBundleData((ResourceBundle)(reqidx[index]->Value),
+
(llevel-aclevel) * ratio,
+
llevel-aclevel);
+ reqcoreleft -= llevel-aclevel;
+
+ elog(RMLOG, "Resource manager acquires %lf GRM
containers on "
+ "host %s. Current
level(having pending) %d, "
+ "expect level %d, acquired
in current request %d.",
+
((ResourceBundle)(reqidx[index]->Value))->Core,
+
GET_SEGRESOURCE_HOSTNAME(((SegResource)(reqidx[index]->Key))),
+ clevel,
+ llevel,
+ aclevel-clevel);
+ }
+
+ index++;
+ }
+ }
+
+ /* Adjust total mem and core request. */
+ *reqcore -= reqcoreleft;
--- End diff --
Yes, it is possible to be negative, for example, 1 GRM container is
required, but in order to level up the minimum water level, 3 segments have to
have 3 more GRM containers added into the request, which should cause
reqcoreleft -2.
I will check infinite loop problem and fix it. Thank you!
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
