wilfred-s commented on a change in pull request #145:
URL: 
https://github.com/apache/incubator-yunikorn-core/pull/145#discussion_r428447717



##########
File path: pkg/scheduler/sorters.go
##########
@@ -47,39 +39,101 @@ func sortQueue(queues []*SchedulingQueue, sortType 
SortType) {
                        return comp < 0
                })
        }
+       metrics.GetSchedulerMetrics().ObserveQueueSortingLatency(sortingStart)
 }
 
-func sortApplications(apps []*SchedulingApplication, sortType SortType, 
globalResource *resources.Resource) {
-       // TODO add latency metric
+func sortApplications(apps map[string]*SchedulingApplication, sortType 
policies.SortPolicy, globalResource *resources.Resource) 
[]*SchedulingApplication {
+       sortingStart := time.Now()
+       var sortedApps []*SchedulingApplication
        switch sortType {
-       case FairSortPolicy:
+       case policies.FairSortPolicy:
+               sortedApps = filterOnPendingResources(apps)
                // Sort by usage
-               sort.SliceStable(apps, func(i, j int) bool {
-                       l := apps[i]
-                       r := apps[j]
+               sort.SliceStable(sortedApps, func(i, j int) bool {
+                       l := sortedApps[i]
+                       r := sortedApps[j]
                        return resources.CompUsageRatio(l.getAssumeAllocated(), 
r.getAssumeAllocated(), globalResource) < 0
                })
-       case FifoSortPolicy:
+       case policies.FifoSortPolicy:
+               sortedApps = filterOnPendingResources(apps)
                // Sort by submission time oldest first
-               sort.SliceStable(apps, func(i, j int) bool {
-                       l := apps[i]
-                       r := apps[j]
+               sort.SliceStable(sortedApps, func(i, j int) bool {
+                       l := sortedApps[i]
+                       r := sortedApps[j]
                        return l.ApplicationInfo.SubmissionTime < 
r.ApplicationInfo.SubmissionTime
                })
+       case policies.StateAwarePolicy:
+               sortedApps = stateAwareFilter(apps)
+               // Sort by submission time oldest first
+               sort.SliceStable(sortedApps, func(i, j int) bool {
+                       l := sortedApps[i]
+                       r := sortedApps[j]
+                       return l.ApplicationInfo.SubmissionTime < 
r.ApplicationInfo.SubmissionTime
+               })
+       }
+       metrics.GetSchedulerMetrics().ObserveAppSortingLatency(sortingStart)
+       return sortedApps
+}
+
+func filterOnPendingResources(apps map[string]*SchedulingApplication) 
[]*SchedulingApplication {
+       filteredApps := make([]*SchedulingApplication, 0)
+       for _, app := range apps {
+               // Only look at app when pending-res > 0
+               if resources.StrictlyGreaterThanZero(app.GetPendingResource()) {
+                       filteredApps = append(filteredApps, app)
+               }
+       }
+       return filteredApps
+}
+
+// This filter only allows one (1) application with a state that is not 
running in the list of candidates.
+// The preference is a state of Starting. If we can not find an app with a 
starting state we will use an app
+// with an Accepted state. However if there is an app with a Starting state 
even with no pending resource
+// requests, no Accepted apps can be scheduled. An app in New state does not 
have any asks and can never be
+// scheduled.
+func stateAwareFilter(apps map[string]*SchedulingApplication) 
[]*SchedulingApplication {

Review comment:
       Which part is a blocker: the gang side is not possible to solve as this 
sorting was never designed to solve gang scheduling. Gang scheduling is also 
not related to the state change that is or is not triggered. A gang would be a 
set of asks to return only if that set is complete. The app could ask once or 
more times for a gang, it might also not be the first request coming in.
   
   Two things:
   I chose the timeout of 5 minutes, no idea if that is too long or not as the 
default.
   
   For the single container/pod we can leverage a tag or label on the 
application. It will not change anything on the filtering side. The only impact 
will be in the app when moving into and out of the `Starting` state. If the tag 
or label is set the app itself quickly auto progresses using a `RunApplication` 
event. We do not skip the state we move the app out of the `Starting` state 
almost immediately. That solves the single container/pod issue.
   
   Requiring or even allowing multiple allocations to move out of `Starting` is 
not really a use case that will show up as far as I can assess. It does not 
cover a gang type request, it does also not fit in with any application type I 
know.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
[email protected]


Reply via email to