This is an automated email from the ASF dual-hosted git repository.
mani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-core.git
The following commit(s) were added to refs/heads/master by this push:
new cfa8a8d4 [YUNIKORN-2570] Add test cases to break the current
preemption flow (#847)
cfa8a8d4 is described below
commit cfa8a8d442974f50fe20757fdd9257412dc0fc0b
Author: Manikandan R <[email protected]>
AuthorDate: Mon May 13 11:37:56 2024 +0530
[YUNIKORN-2570] Add test cases to break the current preemption flow (#847)
Closes: #847
Signed-off-by: Manikandan R <[email protected]>
---
pkg/scheduler/objects/preemption_test.go | 1193 ++++++++++++++++++++++++++++++
1 file changed, 1193 insertions(+)
diff --git a/pkg/scheduler/objects/preemption_test.go
b/pkg/scheduler/objects/preemption_test.go
index 2b0bf7aa..009995e1 100644
--- a/pkg/scheduler/objects/preemption_test.go
+++ b/pkg/scheduler/objects/preemption_test.go
@@ -19,6 +19,7 @@
package objects
import (
+ "strconv"
"testing"
"time"
@@ -29,6 +30,10 @@ import (
"github.com/apache/yunikorn-core/pkg/plugins"
)
+const appID3 = "app-3"
+const alloc = "alloc"
+const node1 = "node1"
+
func TestCheckPreconditions(t *testing.T) {
node := newNode("node1", map[string]resources.Quantity{"first": 5})
iterator := getNodeIteratorFn(node)
@@ -226,6 +231,1194 @@ func TestTryPreemption(t *testing.T) {
assert.Check(t, !alloc2.IsPreempted(), "alloc2 not preempted")
}
+// TestTryPreemptionOnNode Test try preemption on node with simple queue
hierarchy. Since Node doesn't have enough resources to accomodate, preemption
happens because of node resource constraint.
+// Guaranteed and Max resource set on both victim queue path and preemptor
queue path in 2 levels. victim and preemptor queue are siblings.
+// Ask (Preemptor) resource type matches with all resource types of the
victim. But Guaranteed set only on specific resource type. 2 Victims are
available, but 1 should be preempted because further preemption would make
usage go below the guaranteed quota
+// Setup:
+// Nodes are Node1 and Node2. Nodes are full. No space to accommodate the ask.
+// root.parent. Guaranteed set on parent, first: 10
+// root.parent.child1. Guaranteed set, first: 5. 2 Allocations (belongs to
single app) are running. Each Allocation usage is first:5, pods: 1. Total usage
is first:10, pods: 2.
+// root.parent.child2. Guaranteed set, first: 5. Ask of first:5 is waiting for
resources.
+// 1 Allocation on root.parent.child1 should be preempted to free up resources
for ask arrived in root.parent.child2.
+func TestTryPreemptionOnNode(t *testing.T) {
+ t.SkipNow()
+ node1 := newNode("node1", map[string]resources.Quantity{"first": 5,
"pods": 1})
+ node2 := newNode("node2", map[string]resources.Quantity{"first": 5,
"pods": 1})
+ iterator := getNodeIteratorFn(node1, node2)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
map[string]string{"first": "20"}, map[string]string{"first": "10"})
+ assert.NilError(t, err)
+ childQ1, err := createManagedQueueGuaranteed(parentQ, "child1", false,
map[string]string{"first": "10"}, map[string]string{"first": "5"})
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ, "child2", false,
map[string]string{"first": "10"}, map[string]string{"first": "5"})
+ assert.NilError(t, err)
+ app1 := newApplication(appID1, "default", "root.parent.child1")
+ app1.SetQueue(childQ1)
+ childQ1.applications[appID1] = app1
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5, "pods":
1}))
+ ask1.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5, "pods":
1}))
+ ask2.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ alloc1 := NewAllocation("node1", ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node1.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation("node1", ask2)
+ app1.AddAllocation(alloc2)
+ assert.Check(t, node2.AddAllocation(alloc2), "node alloc2 failed")
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+ app2 := newApplication(appID2, "default", "root.parent.child2")
+ app2.SetQueue(childQ2)
+ childQ2.applications[appID2] = app2
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5, "pods":
1}))
+ assert.NilError(t, app2.AddAllocationAsk(ask3))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 10, "pods":
3})
+ preemptor := NewPreemptor(app2, headRoom, 30*time.Second, ask3,
iterator(), false)
+
+ alloc, ok := preemptor.TryPreemption()
+ assert.Assert(t, ok, "no victims found")
+ assert.Equal(t, "alloc3", alloc.allocationKey, "wrong alloc")
+ assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted")
+ assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted")
+}
+
+// TestTryPreemptionOnQueue Test try preemption on queue with simple queue
hierarchy. Since Node has enough resources to accomodate, preemption happens
because of queue resource constraint.
+// Guaranteed and Max resource set on both victim queue path and preemptor
queue path in 2 levels. victim and preemptor queue are siblings.
+// Ask (Preemptor) resource type matches with all resource types of the
victim. But Guaranteed set only on specific resource type. 2 Victims are
available, but 1 should be preempted because further preemption would make
usage go below the guaranteed quota
+// Setup:
+// Nodes are Node1 and Node2. Node has enough space to accommodate the new ask.
+// root.parent. Guaranteed set on parent, first: 10
+// root.parent.child1. Guaranteed set, first: 5. 2 Allocations (belongs to
single app) are running. Each Allocation usage is first:5, pods: 1. Total usage
is first:10, pods: 2.
+// root.parent.child2. Guaranteed set, first: 5. Ask of first:5 is waiting for
resources.
+// 1 Allocation on root.parent.child1 should be preempted to free up resources
for ask arrived in root.parent.child2.
+func TestTryPreemptionOnQueue(t *testing.T) {
+ t.SkipNow()
+ node1 := newNode("node1", map[string]resources.Quantity{"first": 10,
"pods": 2})
+ node2 := newNode("node2", map[string]resources.Quantity{"first": 10,
"pods": 2})
+ iterator := getNodeIteratorFn(node1, node2)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
map[string]string{"first": "10"}, nil)
+ assert.NilError(t, err)
+ childQ1, err := createManagedQueueGuaranteed(parentQ, "child1", false,
nil, map[string]string{"first": "5"})
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ, "child2", false,
nil, map[string]string{"first": "5"})
+ assert.NilError(t, err)
+ app1 := newApplication(appID1, "default", "root.parent.child1")
+ app1.SetQueue(childQ1)
+ childQ1.applications[appID1] = app1
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5, "pods":
1}))
+ ask1.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5, "pods":
1}))
+ ask2.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ alloc1 := NewAllocation("node1", ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node1.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation("node1", ask2)
+ app1.AddAllocation(alloc2)
+ assert.Check(t, node2.AddAllocation(alloc2), "node alloc2 failed")
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+ app2 := newApplication(appID2, "default", "root.parent.child2")
+ app2.SetQueue(childQ2)
+ childQ2.applications[appID2] = app2
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5, "pods":
1}))
+ assert.NilError(t, app2.AddAllocationAsk(ask3))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 10, "pods":
3})
+ preemptor := NewPreemptor(app2, headRoom, 30*time.Second, ask3,
iterator(), false)
+
+ alloc, ok := preemptor.TryPreemption()
+ assert.Assert(t, ok, "no victims found")
+ assert.Equal(t, "alloc3", alloc.allocationKey, "wrong alloc")
+ assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted")
+ assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted")
+}
+
+// TestTryPreemption_VictimsAvailable_InsufficientResource Test try preemption
on queue with simple queue hierarchy. Since Node has enough resources to
accomodate, preemption happens because of queue resource constraint.
+// Guaranteed and Max resource set on both victim queue path and preemptor
queue path. victim and preemptor queue are siblings.
+// Ask (Preemptor) resource type matches with 1 resource type of the victim.
Guaranteed also set on specific resource type. 2 Victims are available, but
total resource usage is lesser than ask requirement.
+// Setup:
+// Nodes are Node1 and Node2. Node has enough space to accommodate the new ask.
+// root.parent.Max set on parent, first: 8
+// root.parent.child1. Guaranteed not set. 2 Allocations (belongs to single
app) are running. Each Allocation usage is first:2, pods: 1. Total usage is
first:4, pods: 2.
+// root.parent.child2. Guaranteed set, first: 5. Ask of first:5 is waiting for
resources.
+// 2 Allocation on root.parent.child1 has been found and considered as
victims. Since victims total resource usage (first: 4) is lesser than ask
requirment (first: 5), preemption won't help. Hence, victims are dropped.
+func TestTryPreemption_VictimsAvailable_InsufficientResource(t *testing.T) {
+ t.SkipNow()
+ node1 := newNode(node1, map[string]resources.Quantity{"first": 10,
"pods": 2})
+ node2 := newNode("node2", map[string]resources.Quantity{"first": 10,
"pods": 2})
+ iterator := getNodeIteratorFn(node1, node2)
+ rootQ, err := createRootQueue(map[string]string{"first": "20", "pods":
"5"})
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
map[string]string{"first": "8"}, nil)
+ assert.NilError(t, err)
+ childQ1, err := createManagedQueueGuaranteed(parentQ, "child1", false,
nil, nil)
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ, "child2", false,
nil, map[string]string{"first": "5"})
+ assert.NilError(t, err)
+ app1 := newApplication(appID1, "default", "root.parent.child1")
+ app1.SetQueue(childQ1)
+ childQ1.applications[appID1] = app1
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 2, "pods":
1}))
+ ask1.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 2, "pods":
1}))
+ ask2.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ alloc1 := NewAllocation("node1", ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node1.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation("node1", ask2)
+ app1.AddAllocation(alloc2)
+ assert.Check(t, node2.AddAllocation(alloc2), "node alloc2 failed")
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+ app2 := newApplication(appID2, "default", "root.parent.child2")
+ app2.SetQueue(childQ2)
+ childQ2.applications[appID2] = app2
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5}))
+ assert.NilError(t, app2.AddAllocationAsk(ask3))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 10, "pods":
3})
+ preemptor := NewPreemptor(app2, headRoom, 30*time.Second, ask3,
iterator(), false)
+
+ alloc, ok := preemptor.TryPreemption()
+ var expectedAlloc *Allocation
+ assert.Equal(t, ok, false, "no victims found")
+ assert.Equal(t, alloc, expectedAlloc, "wrong alloc")
+ assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted")
+ assert.Check(t, !alloc2.IsPreempted(), "alloc2 not preempted")
+}
+
+// TestTryPreemption_VictimsOnDifferentNodes_InsufficientResource Test try
preemption on queue with simple queue hierarchy. Since Node doesn't have enough
resources to accomodate, preemption happens because of node resource constraint.
+// Guaranteed and Max resource set on both victim queue path and preemptor
queue path. victim and preemptor queue are siblings.
+// Ask (Preemptor) resource type matches with 1 resource type of the victim.
Guaranteed also set on specific resource type. 2 Victims are available, but
total resource usage is lesser than ask requirement.
+// Setup:
+// Nodes are Node1 and Node2. Both Nodes are almost full. No space to
accommodate the ask.
+// root.parent.Max set on parent, first: 6
+// root.parent.child1. Guaranteed not set. 2 Allocations (belongs to single
app) are running. Each Allocation usage is first:2, pods: 1. Total usage is
first:4, pods: 2.
+// root.parent.child2. Guaranteed set, first: 5. Ask of first:5 is waiting for
resources.
+// 2 Allocation on root.parent.child1 has been found and considered as
victims. Since victims total resource usage (first: 4) is lesser than ask
requirment (first: 5), preemption won't help. Hence, victims are dropped.
+func TestTryPreemption_VictimsOnDifferentNodes_InsufficientResource(t
*testing.T) {
+ t.SkipNow()
+ node1 := newNode(node1, map[string]resources.Quantity{"first": 5,
"pods": 1})
+ node2 := newNode("node2", map[string]resources.Quantity{"first": 5,
"pods": 1})
+ iterator := getNodeIteratorFn(node1, node2)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
map[string]string{"first": "6"}, nil)
+ assert.NilError(t, err)
+ childQ1, err := createManagedQueueGuaranteed(parentQ, "child1", false,
nil, nil)
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ, "child2", false,
nil, map[string]string{"first": "5"})
+ assert.NilError(t, err)
+ app1 := newApplication(appID1, "default", "root.parent.child1")
+ app1.SetQueue(childQ1)
+ childQ1.applications[appID1] = app1
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 2, "pods":
1}))
+ ask1.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 2, "pods":
1}))
+ ask2.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ alloc1 := NewAllocation("node1", ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node1.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation("node2", ask2)
+ app1.AddAllocation(alloc2)
+ assert.Check(t, node2.AddAllocation(alloc2), "node alloc2 failed")
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+ app2 := newApplication(appID2, "default", "root.parent.child2")
+ app2.SetQueue(childQ2)
+ childQ2.applications[appID2] = app2
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5}))
+ assert.NilError(t, app2.AddAllocationAsk(ask3))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 10, "pods":
3})
+ preemptor := NewPreemptor(app2, headRoom, 30*time.Second, ask3,
iterator(), false)
+
+ // register predicate handler
+ preemptions := []mock.Preemption{
+ mock.NewPreemption(true, "alloc3", "node1", []string{"alloc1"},
0, 0),
+ mock.NewPreemption(true, "alloc3", "node2", []string{"alloc2"},
0, 0),
+ }
+
+ plugin := mock.NewPreemptionPredicatePlugin(nil, nil, preemptions)
+ plugins.RegisterSchedulerPlugin(plugin)
+ defer plugins.UnregisterSchedulerPlugins()
+
+ alloc, ok := preemptor.TryPreemption()
+ var expectedAlloc *Allocation
+ assert.Equal(t, ok, false, "no victims found")
+ assert.Equal(t, alloc, expectedAlloc, "wrong alloc")
+ assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted")
+ assert.Check(t, !alloc2.IsPreempted(), "alloc2 not preempted")
+}
+
+// TestTryPreemption_VictimsAvailableOnDifferentNodes Test try preemption on
queue with simple queue hierarchy. Since Node doesn't have enough resources to
accomodate, preemption happens because of node resource constraint.
+// Guaranteed and Max resource set on both victim queue path and preemptor
queue path. victim and preemptor queue are siblings.
+// Ask (Preemptor) resource type matches with 1 resource type of the victim.
Guaranteed also set on specific resource type. 2 Victims are available, but
total resource usage is lesser than ask requirement.
+// Setup:
+// Nodes are Node1 and Node2. Both Nodes are almost full. No space to
accommodate the ask. Node 2 won't even fit because max capacity itself is
lesser than ask requirement. Hence, Node 1 only is eligible.
+// root.parent.Max set on parent, first: 6
+// root.parent.child1. Guaranteed not set. 2 Allocations (belongs to single
app) are running. 1st Allocation usage is first:4, pods: 1. 2nd Allocation
usage is first:2, pods: 1. Total usage is first:6, pods: 2.
+// root.parent.child2. Guaranteed set, first: 5. Ask of first:5 is waiting for
resources.
+// 2 Allocation on root.parent.child1 has been found and considered as victims
and preempted to free up resources for ask.
+func TestTryPreemption_VictimsAvailableOnDifferentNodes(t *testing.T) {
+ t.SkipNow()
+ node1 := newNode(node1, map[string]resources.Quantity{"first": 5,
"pods": 1})
+ node2 := newNode("node2", map[string]resources.Quantity{"first": 4,
"pods": 1})
+ iterator := getNodeIteratorFn(node1, node2)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
map[string]string{"first": "6"}, nil)
+ assert.NilError(t, err)
+ childQ1, err := createManagedQueueGuaranteed(parentQ, "child1", false,
nil, nil)
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ, "child2", false,
nil, map[string]string{"first": "5"})
+ assert.NilError(t, err)
+ app1 := newApplication(appID1, "default", "root.parent.child1")
+ app1.SetQueue(childQ1)
+ childQ1.applications[appID1] = app1
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 4, "pods":
1}))
+ ask1.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 2, "pods":
1}))
+ ask2.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ alloc1 := NewAllocation("node1", ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node1.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation("node2", ask2)
+ app1.AddAllocation(alloc2)
+ assert.Check(t, node2.AddAllocation(alloc2), "node alloc2 failed")
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+ app2 := newApplication(appID2, "default", "root.parent.child2")
+ app2.SetQueue(childQ2)
+ childQ2.applications[appID2] = app2
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5}))
+ assert.NilError(t, app2.AddAllocationAsk(ask3))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 10, "pods":
3})
+ preemptor := NewPreemptor(app2, headRoom, 30*time.Second, ask3,
iterator(), false)
+
+ // register predicate handler
+ preemptions := []mock.Preemption{
+ mock.NewPreemption(true, "alloc3", "node1", []string{"alloc1"},
0, 0),
+ mock.NewPreemption(true, "alloc3", "node2", []string{"alloc2"},
0, 0),
+ }
+
+ plugin := mock.NewPreemptionPredicatePlugin(nil, nil, preemptions)
+ plugins.RegisterSchedulerPlugin(plugin)
+ defer plugins.UnregisterSchedulerPlugins()
+
+ alloc, ok := preemptor.TryPreemption()
+ assert.Assert(t, ok, "no victims found")
+ assert.Equal(t, "alloc3", alloc.allocationKey, "wrong alloc")
+ assert.Check(t, alloc1.IsPreempted(), "alloc1 preempted")
+ assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted")
+}
+
+// TestTryPreemption_OnQueue_VictimsOnDifferentNodes Test try preemption on
queue with simple queue hierarchy. Since Node has enough resources to
accomodate, preemption happens because of queue resource constraint.
+// Guaranteed and Max resource set on both victim queue path and preemptor
queue paths. victim and preemptor queue are siblings.
+// Ask (Preemptor) resource type matches with all resource types of the
victim. Guaranteed set only on that specific resource type.
+// Setup:
+// Nodes are Node1 and Node2. Node has enough space to accommodate the new ask.
+// root.parent. Max set on parent, first: 18
+// root.parent.child1. Guaranteed not set. 2 Allocations (belongs to single
app) are running on node1 and node2. Each Allocation usage is first:5. Total
usage is first:10.
+// root.parent.child2. Guaranteed set 5. Ask of first:5 is waiting for
resources.
+// root.parent.child3. Guaranteed not set. 1 Allocation is running on node2.
Total usage is first:5.
+// Preemption options are 1. 2 Alloc running on Node 2 but on child 1 and
child 3 queues. 2. 2 Alloc running on Node 2 and child 1 queue. 3. All three 3
allocs.
+// option 1 >> option 2 >> option 3. In option 3, preempting third allocation
is unnecessary, should avoid this option.
+// Either option 1 or option2 is fine, but not option 3.
+func TestTryPreemption_OnQueue_VictimsOnDifferentNodes(t *testing.T) {
+ t.SkipNow()
+ node1 := newNode(node1, map[string]resources.Quantity{"first": 30})
+ node2 := newNode("node2", map[string]resources.Quantity{"first": 30})
+ iterator := getNodeIteratorFn(node1, node2)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
map[string]string{"first": "18"}, nil)
+ assert.NilError(t, err)
+ childQ1, err := createManagedQueueGuaranteed(parentQ, "child1", false,
nil, nil)
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ, "child2", false,
nil, map[string]string{"first": "15"})
+ assert.NilError(t, err)
+ childQ3, err := createManagedQueueGuaranteed(parentQ, "child3", false,
nil, nil)
+ assert.NilError(t, err)
+ app1 := newApplication(appID1, "default", "root.parent.child1")
+ app1.SetQueue(childQ1)
+ childQ1.applications[appID1] = app1
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5}))
+ ask1.createTime = time.Now().Add(-2 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5}))
+ ask2.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ alloc1 := NewAllocation("node1", ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node1.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation("node2", ask2)
+ app1.AddAllocation(alloc2)
+ assert.Check(t, node2.AddAllocation(alloc2), "node alloc2 failed")
+
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+
+ app3 := newApplication(appID3, "default", "root.parent.child3")
+ app3.SetQueue(childQ3)
+ childQ3.applications[appID3] = app3
+
+ ask4 := newAllocationAsk("alloc4", appID3,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5}))
+ ask4.createTime = time.Now()
+ assert.NilError(t, app3.AddAllocationAsk(ask4))
+
+ alloc4 := NewAllocation("node2", ask4)
+ app3.AddAllocation(alloc4)
+ assert.Check(t, node2.AddAllocation(alloc4), "node alloc2 failed")
+ assert.NilError(t,
childQ3.IncAllocatedResource(ask4.GetAllocatedResource(), false))
+
+ app2 := newApplication(appID2, "default", "root.parent.child2")
+ app2.SetQueue(childQ2)
+ childQ2.applications[appID2] = app2
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 10}))
+ assert.NilError(t, app2.AddAllocationAsk(ask3))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 10, "pods":
3})
+ preemptor := NewPreemptor(app2, headRoom, 30*time.Second, ask3,
iterator(), false)
+
+ allocs := map[string]string{}
+ allocs["alloc3"] = "node2"
+
+ plugin := mock.NewPreemptionPredicatePlugin(nil, allocs, nil)
+ plugins.RegisterSchedulerPlugin(plugin)
+ defer plugins.UnregisterSchedulerPlugins()
+
+ alloc, ok := preemptor.TryPreemption()
+ assert.Assert(t, ok, "no victims found")
+ assert.Equal(t, "alloc3", alloc.allocationKey, "wrong alloc")
+ assert.Equal(t, "node2", alloc.nodeID, "wrong alloc")
+ assert.Equal(t, "node1", alloc1.nodeID, "wrong alloc")
+ assert.Equal(t, "node2", alloc2.nodeID, "wrong alloc")
+ assert.Equal(t, "node2", alloc4.nodeID, "wrong alloc")
+ assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted")
+ assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted")
+ assert.Check(t, alloc4.IsPreempted(), "alloc2 not preempted")
+}
+
+// TestTryPreemption_OnQueue_VictimsAvailable_LowerPriority Test try
preemption on queue with simple queue hierarchy. Since Node has enough
resources to accomodate, preemption happens because of queue resource
constraint.
+// Guaranteed and Max resource set on both victim queue path and preemptor
queue paths. victim and preemptor queue are siblings.
+// Ask (Preemptor) resource type matches with all resource types of the
victim. Guaranteed set only on that specific resource type.
+// Setup:
+// Nodes are Node1 and Node2. Node has enough space to accommodate the new ask.
+// root.parent. Max set on parent, first: 18
+// root.parent.child1. Guaranteed not set. 2 Allocations (belongs to single
app) are running on node1 and node2. Each Allocation usage is first:5. Total
usage is first:10.
+// root.parent.child2. Guaranteed set 5. Ask of first:5 is waiting for
resources.
+// root.parent.child3. Guaranteed not set. 1 Allocation is running on node2.
Total usage is first:5.
+// High priority ask should not be touched and remaining 2 allocs should be
preempted to free up resources
+func TestTryPreemption_OnQueue_VictimsAvailable_LowerPriority(t *testing.T) {
+ t.SkipNow()
+ node1 := newNode(node1, map[string]resources.Quantity{"first": 30})
+ node2 := newNode("node2", map[string]resources.Quantity{"first": 30})
+ iterator := getNodeIteratorFn(node1, node2)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
map[string]string{"first": "18"}, nil)
+ assert.NilError(t, err)
+ childQ1, err := createManagedQueueGuaranteed(parentQ, "child1", false,
nil, nil)
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ, "child2", false,
nil, map[string]string{"first": "15"})
+ assert.NilError(t, err)
+ childQ3, err := createManagedQueueGuaranteed(parentQ, "child3", false,
nil, nil)
+ assert.NilError(t, err)
+ app1 := newApplication(appID1, "default", "root.parent.child1")
+ app1.SetQueue(childQ1)
+ childQ1.applications[appID1] = app1
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5}))
+ ask1.createTime = time.Now().Add(-2 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+
+ // High priority ask, should not be considered as victim
+ ask2 := newAllocationAskPriority("alloc2", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5}), 1000)
+ ask2.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ alloc1 := NewAllocation("node1", ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node1.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation("node2", ask2)
+ app1.AddAllocation(alloc2)
+ assert.Check(t, node2.AddAllocation(alloc2), "node alloc2 failed")
+
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ1.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+
+ app3 := newApplication(appID3, "default", "root.parent.child3")
+ app3.SetQueue(childQ3)
+ childQ3.applications[appID3] = app3
+
+ ask4 := newAllocationAsk("alloc4", appID3,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5}))
+ ask4.createTime = time.Now()
+ assert.NilError(t, app3.AddAllocationAsk(ask4))
+
+ alloc4 := NewAllocation("node2", ask4)
+ app3.AddAllocation(alloc4)
+ assert.Check(t, node2.AddAllocation(alloc4), "node alloc2 failed")
+ assert.NilError(t,
childQ3.IncAllocatedResource(ask4.GetAllocatedResource(), false))
+
+ app2 := newApplication(appID2, "default", "root.parent.child2")
+ app2.SetQueue(childQ2)
+ childQ2.applications[appID2] = app2
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 10}))
+ assert.NilError(t, app2.AddAllocationAsk(ask3))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 10, "pods":
3})
+ preemptor := NewPreemptor(app2, headRoom, 30*time.Second, ask3,
iterator(), false)
+
+ allocs := map[string]string{}
+ allocs["alloc3"] = "node2"
+
+ plugin := mock.NewPreemptionPredicatePlugin(nil, allocs, nil)
+ plugins.RegisterSchedulerPlugin(plugin)
+ defer plugins.UnregisterSchedulerPlugins()
+
+ alloc, ok := preemptor.TryPreemption()
+ assert.Assert(t, ok, "no victims found")
+ assert.Equal(t, "alloc3", alloc.allocationKey, "wrong alloc")
+ assert.Equal(t, "node2", alloc.nodeID, "wrong alloc")
+ assert.Equal(t, "node1", alloc1.nodeID, "wrong alloc")
+ assert.Equal(t, "node2", alloc2.nodeID, "wrong alloc")
+ assert.Equal(t, "node2", alloc4.nodeID, "wrong alloc")
+ assert.Check(t, alloc1.IsPreempted(), "alloc1 preempted")
+ assert.Check(t, !alloc2.IsPreempted(), "alloc2 not preempted")
+ assert.Check(t, alloc4.IsPreempted(), "alloc2 not preempted")
+}
+
+// TestTryPreemption_AskResTypesDifferent_GuaranteedSetOnPreemptorSide Test
try preemption with 2 level queue hierarchy.
+// Guaranteed set only on preemptor queue path, but not on the victim queue
path.
+// Ask (Preemptor) resource type matches with one of the victim's resource
types. Still, needs to be preempted because matching resource type has been
configured as guaranteed.
+// Setup:
+// Nodes are Node1. Node has enough space to accommodate the new ask.
+// root.parent.parent1.child1. Guaranteed set on root.parent.parent1, vcores:
1. Ask of vcores: 1 is waiting for resources.
+// root.parent.parent2.child2. 2 Allocations (belongs to single app) are
running. Each Allocation usage is vcores:1, mem: 200. Total usage is vcores:2,
mem: 400
+// root.parent.parent2.child3. No usage, no guaranteed set
+// 1 Allocation on root.parent.parent1.child2 should be preempted to free up
resources for ask arrived in root.parent.parent1.child1.
+func TestTryPreemption_AskResTypesDifferent_GuaranteedSetOnPreemptorSide(t
*testing.T) {
+ t.SkipNow()
+ node := newNode(node1, map[string]resources.Quantity{"vcores": 3,
"mem": 400})
+ iterator := getNodeIteratorFn(node)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
map[string]string{"vcores": "2"}, nil)
+ assert.NilError(t, err)
+ parentQ1, err := createManagedQueueGuaranteed(parentQ, "parent1", true,
nil, map[string]string{"vcores": "1"})
+ assert.NilError(t, err)
+ parentQ2, err := createManagedQueueGuaranteed(parentQ, "parent2", true,
nil, nil)
+ assert.NilError(t, err)
+
+ childQ1, err := createManagedQueueGuaranteed(parentQ1, "child1", false,
nil, nil)
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ2, "child2", false,
nil, nil)
+ assert.NilError(t, err)
+ _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil,
nil)
+ assert.NilError(t, err)
+
+ app1 := newApplication(appID1, "default", "root.parent.parent2.child2")
+ app1.SetQueue(childQ2)
+ childQ2.applications[appID1] = app1
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1, "mem":
200}))
+ ask1.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1, "mem":
200}))
+ ask2.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ alloc1 := NewAllocation(node1, ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation(node1, ask2)
+ app1.AddAllocation(alloc2)
+ assert.Check(t, node.AddAllocation(alloc2), "node alloc2 failed")
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+ app2 := newApplication(appID2, "default", "root.parent.parent1.child1")
+ app2.SetQueue(childQ1)
+ childQ1.applications[appID2] = app2
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ assert.NilError(t, app2.AddAllocationAsk(ask3))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2})
+ preemptor := NewPreemptor(app2, headRoom, 30*time.Second, ask3,
iterator(), false)
+
+ alloc, ok := preemptor.TryPreemption()
+ assert.Assert(t, ok, "no victims found")
+ assert.Equal(t, "alloc3", alloc.allocationKey, "wrong alloc")
+ assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted")
+ assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted")
+}
+
+// TestTryPreemption_OnNode_AskResTypesDifferent_GuaranteedSetOnPreemptorSide
Test try preemption with 2 level queue hierarchy. Since Node doesn't have
enough resources to accomodate, preemption happens because of node resource
constraint.
+// Guaranteed set only on preemptor queue path, but not on the victim queue
path.
+// Ask (Preemptor) resource type matches with one of the victim's resource
types. Still, needs to be preempted because matching resource type has been
configured as guaranteed.
+// Setup:
+// Nodes are Node1. Node is full, doesn't enough space to accommodate the ask.
+// root.parent.parent1.child1. Guaranteed set on root.parent.parent1, vcores:
1. Ask of vcores: 1 is waiting for resources.
+// root.parent.parent2.child2. 2 Allocations (belongs to single app) are
running. Each Allocation usage is vcores:1, mem: 200. Total usage is vcores:2,
mem: 400
+// root.parent.parent2.child3. No usage, no guaranteed set
+// 1 Allocation on root.parent.parent1.child2 should be preempted to free up
resources for ask arrived in root.parent.parent1.child1.
+func
TestTryPreemption_OnNode_AskResTypesDifferent_GuaranteedSetOnPreemptorSide(t
*testing.T) {
+ t.SkipNow()
+ node := newNode(node1, map[string]resources.Quantity{"vcores": 2,
"mem": 400})
+ iterator := getNodeIteratorFn(node)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
map[string]string{"vcores": "2"}, nil)
+ assert.NilError(t, err)
+ parentQ1, err := createManagedQueueGuaranteed(parentQ, "parent1", true,
nil, map[string]string{"vcores": "1"})
+ assert.NilError(t, err)
+ parentQ2, err := createManagedQueueGuaranteed(parentQ, "parent2", true,
nil, nil)
+ assert.NilError(t, err)
+
+ childQ1, err := createManagedQueueGuaranteed(parentQ1, "child1", false,
nil, nil)
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ2, "child2", false,
nil, nil)
+ assert.NilError(t, err)
+ _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil,
nil)
+ assert.NilError(t, err)
+
+ app1 := newApplication(appID1, "default", "root.parent.parent2.child2")
+ app1.SetQueue(childQ2)
+ childQ2.applications[appID1] = app1
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1, "mem":
200}))
+ ask1.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1, "mem":
200}))
+ ask2.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ alloc1 := NewAllocation(node1, ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation(node1, ask2)
+ app1.AddAllocation(alloc2)
+ assert.Check(t, node.AddAllocation(alloc2), "node alloc2 failed")
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+ app2 := newApplication(appID2, "default", "root.parent.parent1.child1")
+ app2.SetQueue(childQ1)
+ childQ1.applications[appID2] = app2
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ assert.NilError(t, app2.AddAllocationAsk(ask3))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2})
+ preemptor := NewPreemptor(app2, headRoom, 30*time.Second, ask3,
iterator(), false)
+
+ // register predicate handler
+ preemptions := []mock.Preemption{mock.NewPreemption(true, "alloc3",
node1, []string{"alloc2"}, 0, 0)}
+ plugin := mock.NewPreemptionPredicatePlugin(nil, nil, preemptions)
+ plugins.RegisterSchedulerPlugin(plugin)
+ defer plugins.UnregisterSchedulerPlugins()
+
+ alloc, ok := preemptor.TryPreemption()
+ assert.Assert(t, ok, "no victims found")
+ assert.Equal(t, "alloc3", alloc.allocationKey, "wrong alloc")
+ assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted")
+ assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted")
+}
+
+//
TestTryPreemption_AskResTypesDifferent_GuaranteedSetOnVictimAndPreemptorSides
Test try preemption with 2 level queue hierarchy.
+// Guaranteed set on both victim queue path and preemptor queue path.
+// Ask (Preemptor) resource type matches with one of the victim's resource
types. Still, needs to be preempted because matching resource type has been
configured as guaranteed.
+// Setup:
+// Nodes are Node1. Node has enough space to accommodate the new ask.
+// root.parent.parent1.child1. Guaranteed set on parent1, vcores: 2. Ask of
vcores: 1 is waiting for resources.
+// root.parent.parent2.child2. Guaranteed set on parent2, vcores: 1. 6
Allocations (belongs to two diff apps) are running. 3 Allocation's usage is
vcores:1, mem: 100. 3 more allocations usage is mem: 100. Total usage is
vcores:3, mem: 600
+// root.parent.parent2.child3. No usage, no guaranteed set
+// 2 Allocations of each vcores:1, mem: 100 running on
root.parent.parent1.child2 should be preempted to free up resources for ask
arrived in root.parent.parent1.child1.
+// 3rd allocation of vcores:1, mem: 100 should not be touched as preempting
the same would make usage goes below the guaranteed set on
root.parent.parent2.child2.
+// All remaining three allocation of each mem: 100 should not be touched at
all as there is no matching resource type between these allocs and ask resource
types.
+func
TestTryPreemption_AskResTypesDifferent_GuaranteedSetOnVictimAndPreemptorSides(t
*testing.T) {
+ t.SkipNow()
+ node := newNode(node1, map[string]resources.Quantity{"vcores": 5,
"mem": 700})
+ iterator := getNodeIteratorFn(node)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
map[string]string{"vcores": "3"}, nil)
+ assert.NilError(t, err)
+ parentQ1, err := createManagedQueueGuaranteed(parentQ, "parent1", true,
nil, nil)
+ assert.NilError(t, err)
+ parentQ2, err := createManagedQueueGuaranteed(parentQ, "parent2", true,
nil, nil)
+ assert.NilError(t, err)
+
+ childQ1, err := createManagedQueueGuaranteed(parentQ1, "child1", false,
nil, map[string]string{"vcores": "2"})
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ2, "child2", false,
nil, map[string]string{"vcores": "1"})
+ assert.NilError(t, err)
+ _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil,
nil)
+ assert.NilError(t, err)
+
+ app1 := newApplication(appID1, "default", "root.parent.parent2.child2")
+ app1.SetQueue(childQ2)
+ childQ2.applications[appID1] = app1
+ app2 := newApplication(appID2, "default", "root.parent.parent2.child2")
+ app2.SetQueue(childQ2)
+ childQ2.applications[appID2] = app2
+ app3 := newApplication(appID3, "default", "root.parent.parent2.child2")
+ app3.SetQueue(childQ2)
+ childQ2.applications[appID3] = app3
+
+ for i := 5; i < 8; i++ {
+ askN := newAllocationAsk(alloc+strconv.Itoa(i), appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"mem": 100}))
+ askN.createTime = time.Now().Add(-2 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(askN))
+ allocN := NewAllocation(node1, askN)
+ app1.AddAllocation(allocN)
+ assert.Check(t, node.AddAllocation(allocN), "node alloc1
failed")
+ }
+
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1, "mem":
100}))
+ ask1.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1, "mem":
100}))
+ ask2.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1, "mem":
100}))
+ ask3.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask3))
+ alloc1 := NewAllocation(node1, ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation(node1, ask2)
+ app2.AddAllocation(alloc2)
+ assert.Check(t, node.AddAllocation(alloc2), "node alloc2 failed")
+ alloc3 := NewAllocation(node1, ask3)
+ app3.AddAllocation(alloc3)
+ assert.Check(t, node.AddAllocation(alloc3), "node alloc3 failed")
+
+ for i := 5; i < 8; i++ {
+ assert.NilError(t,
childQ2.IncAllocatedResource(resources.NewResourceFromMap(map[string]resources.Quantity{"mem":
100}), false))
+ }
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask3.GetAllocatedResource(), false))
+
+ app4 := newApplication("app-4", "default", "root.parent.parent1.child1")
+ app4.SetQueue(childQ1)
+ ask4 := newAllocationAsk("alloc4", "app-4",
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2}))
+ assert.NilError(t, app4.AddAllocationAsk(ask4))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2})
+ preemptor := NewPreemptor(app4, headRoom, 30*time.Second, ask4,
iterator(), false)
+
+ // register predicate handler
+ preemptions := []mock.Preemption{mock.NewPreemption(true, "alloc4",
node1, []string{"alloc3", "alloc2"}, 1, 1)}
+ allocs := map[string]string{}
+ allocs["alloc4"] = node1
+ plugin := mock.NewPreemptionPredicatePlugin(nil, allocs, preemptions)
+ plugins.RegisterSchedulerPlugin(plugin)
+ defer plugins.UnregisterSchedulerPlugins()
+
+ alloc, ok := preemptor.TryPreemption()
+ assert.NilError(t, plugin.GetPredicateError())
+ assert.Assert(t, ok, "no victims found")
+ assert.Equal(t, "alloc4", alloc.allocationKey, "wrong alloc")
+ assert.Check(t, !alloc1.IsPreempted(), "alloc1 not preempted")
+ assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted")
+ assert.Check(t, alloc3.IsPreempted(), "alloc3 preempted")
+}
+
+//
TestTryPreemption_OnNode_AskResTypesDifferent_GuaranteedSetOnVictimAndPreemptorSides
Test try preemption with 2 level queue hierarchy. Since Node doesn't have
enough resources to accomodate, preemption happens because of node resource
constraint.
+// Guaranteed set on both victim queue path and preemptor queue path.
+// Ask (Preemptor) resource type matches with one of the victim's resource
types. Still, needs to be preempted because matching resource type has been
configured as guaranteed.
+// Setup:
+// Nodes are Node1. Node doesn't have space. Not able to accommodate the ask.
+// root.parent.parent1.child1. Guaranteed set on parent1, vcores: 2. Ask of
vcores: 1 is waiting for resources.
+// root.parent.parent2.child2. Guaranteed set on parent2, vcores: 1. 6
Allocations (belongs to two diff apps) are running. 3 Allocation's usage is
vcores:1, mem: 100. 3 more allocations usage is mem: 100. Total usage is
vcores:3, mem: 600
+// root.parent.parent2.child3. No usage, no guaranteed set
+// 2 Allocations of each vcores:1, mem: 100 running on
root.parent.parent1.child2 should be preempted to free up resources for ask
arrived in root.parent.parent1.child1.
+// 3rd allocation of vcores:1, mem: 100 should not be touched as preempting
the same would make usage goes below the guaranteed set on
root.parent.parent2.child2.
+// All remaining three allocation of each mem: 100 should not be touched at
all as there is no matching resource type between these allocs and ask resource
types.
+func
TestTryPreemption_OnNode_AskResTypesDifferent_GuaranteedSetOnVictimAndPreemptorSides(t
*testing.T) {
+ t.SkipNow()
+ node := newNode(node1, map[string]resources.Quantity{"vcores": 3,
"mem": 600})
+ iterator := getNodeIteratorFn(node)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
map[string]string{"vcores": "3"}, nil)
+ assert.NilError(t, err)
+ parentQ1, err := createManagedQueueGuaranteed(parentQ, "parent1", true,
nil, nil)
+ assert.NilError(t, err)
+ parentQ2, err := createManagedQueueGuaranteed(parentQ, "parent2", true,
nil, nil)
+ assert.NilError(t, err)
+
+ childQ1, err := createManagedQueueGuaranteed(parentQ1, "child1", false,
nil, map[string]string{"vcores": "2"})
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ2, "child2", false,
nil, map[string]string{"vcores": "1"})
+ assert.NilError(t, err)
+ _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil,
nil)
+ assert.NilError(t, err)
+
+ app1 := newApplication(appID1, "default", "root.parent.parent2.child2")
+ app1.SetQueue(childQ2)
+ childQ2.applications[appID1] = app1
+ app2 := newApplication(appID2, "default", "root.parent.parent2.child2")
+ app2.SetQueue(childQ2)
+ childQ2.applications[appID2] = app2
+ app3 := newApplication(appID3, "default", "root.parent.parent2.child2")
+ app3.SetQueue(childQ2)
+ childQ2.applications[appID3] = app3
+
+ for i := 5; i < 8; i++ {
+ askN := newAllocationAsk(alloc+strconv.Itoa(i), appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"mem": 100}))
+ askN.createTime = time.Now().Add(-2 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(askN))
+ allocN := NewAllocation(node1, askN)
+ app1.AddAllocation(allocN)
+ assert.Check(t, node.AddAllocation(allocN), "node alloc1
failed")
+ }
+
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1, "mem":
100}))
+ ask1.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1, "mem":
100}))
+ ask2.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1, "mem":
100}))
+ ask3.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask3))
+ alloc1 := NewAllocation(node1, ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation(node1, ask2)
+ app2.AddAllocation(alloc2)
+ assert.Check(t, node.AddAllocation(alloc2), "node alloc2 failed")
+ alloc3 := NewAllocation(node1, ask3)
+ app3.AddAllocation(alloc3)
+ assert.Check(t, node.AddAllocation(alloc3), "node alloc3 failed")
+
+ for i := 5; i < 8; i++ {
+ assert.NilError(t,
childQ2.IncAllocatedResource(resources.NewResourceFromMap(map[string]resources.Quantity{"mem":
100}), false))
+ }
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask3.GetAllocatedResource(), false))
+
+ app4 := newApplication("app-4", "default", "root.parent.parent1.child1")
+ app4.SetQueue(childQ1)
+ ask4 := newAllocationAsk("alloc4", "app-4",
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2}))
+ assert.NilError(t, app4.AddAllocationAsk(ask4))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2})
+ preemptor := NewPreemptor(app4, headRoom, 30*time.Second, ask4,
iterator(), false)
+
+ // register predicate handler
+ preemptions := []mock.Preemption{mock.NewPreemption(true, "alloc4",
node1, []string{"alloc3", "alloc2"}, 1, 1)}
+ plugin := mock.NewPreemptionPredicatePlugin(nil, nil, preemptions)
+ plugins.RegisterSchedulerPlugin(plugin)
+ defer plugins.UnregisterSchedulerPlugins()
+
+ alloc, ok := preemptor.TryPreemption()
+ assert.NilError(t, plugin.GetPredicateError())
+ assert.Assert(t, ok, "no victims found")
+ assert.Equal(t, "alloc4", alloc.allocationKey, "wrong alloc")
+ assert.Check(t, !alloc1.IsPreempted(), "alloc1 not preempted")
+ assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted")
+ assert.Check(t, alloc3.IsPreempted(), "alloc3 preempted")
+}
+
+// TestTryPreemption_AskResTypesSame_GuaranteedSetOnPreemptorSide Test try
preemption with 2 level queue hierarchy. Since Node doesn't have enough
resources to accomodate, preemption happens because of node resource constraint.
+// Guaranteed set only on preemptor queue path.
+// Ask (Preemptor) resource type matches with one of the victim's resource
types. Still, needs to be preempted because matching resource type has been
configured as guaranteed.
+// Setup:
+// Nodes are Node1. Node has enough space to accommodate the new ask.
+// root.parent.parent1.child1. Guaranteed set on parent1, vcores: 2. Ask of
vcores: 2, mem: 200 is waiting for resources.
+// root.parent.parent2.child2. Guaranteed set on parent2, vcores: 1. 6
Allocations (belongs to two diff apps) are running. 3 Allocation's usage is
vcores:1. 3 more allocations usage gpu: 100. Total usage is vcores:3, gpu: 300
+// root.parent.parent2.child3. No usage, no guaranteed set
+// 3 Allocations of each vcores:1 running on root.parent.parent1.child2 could
be preempted to free up resources for ask arrived in root.parent.parent1.child1.
+// but last allocation should not be touched as preempting the same would make
usage goes above the guaranteed set on preemptor or ask queue
root.parent.parent2.child1.
+// All remaining three allocation of each mem: 100 should not be touched at
all as there is no matching resource type between these allocs and ask resource
types.
+func TestTryPreemption_AskResTypesSame_GuaranteedSetOnPreemptorSide(t
*testing.T) {
+ t.SkipNow()
+ node := newNode(node1, map[string]resources.Quantity{"vcores": 5,
"gpu": 300, "mem": 200})
+ iterator := getNodeIteratorFn(node)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
nil, nil)
+ assert.NilError(t, err)
+ parentQ1, err := createManagedQueueGuaranteed(parentQ, "parent1", true,
nil, nil)
+ assert.NilError(t, err)
+ parentQ2, err := createManagedQueueGuaranteed(parentQ, "parent2", true,
nil, nil)
+ assert.NilError(t, err)
+
+ childQ1, err := createManagedQueueGuaranteed(parentQ1, "child1", false,
nil, map[string]string{"vcores": "2"})
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ2, "child2", false,
nil, nil)
+ assert.NilError(t, err)
+ _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil,
nil)
+ assert.NilError(t, err)
+
+ app1 := newApplication(appID1, "default", "root.parent.parent2.child2")
+ app1.SetQueue(childQ2)
+ childQ2.applications[appID1] = app1
+ app2 := newApplication(appID2, "default", "root.parent.parent2.child2")
+ app2.SetQueue(childQ2)
+ childQ2.applications[appID2] = app2
+ app3 := newApplication(appID3, "default", "root.parent.parent2.child2")
+ app3.SetQueue(childQ2)
+ childQ2.applications[appID3] = app3
+
+ for i := 5; i < 8; i++ {
+ askN := newAllocationAsk(alloc+strconv.Itoa(i), appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"gpu": 100}))
+ askN.createTime = time.Now().Add(-2 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(askN))
+ allocN := NewAllocation(node1, askN)
+ app1.AddAllocation(allocN)
+ assert.Check(t, node.AddAllocation(allocN), "node alloc1
failed")
+ }
+
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ ask1.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ ask2.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ ask3.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask3))
+ alloc1 := NewAllocation(node1, ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation(node1, ask2)
+ app2.AddAllocation(alloc2)
+ assert.Check(t, node.AddAllocation(alloc2), "node alloc2 failed")
+ alloc3 := NewAllocation(node1, ask3)
+ app3.AddAllocation(alloc3)
+ assert.Check(t, node.AddAllocation(alloc3), "node alloc3 failed")
+
+ for i := 5; i < 8; i++ {
+ assert.NilError(t,
childQ2.IncAllocatedResource(resources.NewResourceFromMap(map[string]resources.Quantity{"gpu":
100}), false))
+ }
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask3.GetAllocatedResource(), false))
+
+ app4 := newApplication("app-4", "default", "root.parent.parent1.child1")
+ app4.SetQueue(childQ1)
+ ask4 := newAllocationAsk("alloc4", "app-4",
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2, "mem":
200}))
+ assert.NilError(t, app4.AddAllocationAsk(ask4))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2})
+ preemptor := NewPreemptor(app4, headRoom, 30*time.Second, ask4,
iterator(), false)
+
+ // register predicate handler
+ preemptions := []mock.Preemption{mock.NewPreemption(true, "alloc4",
node1, []string{"alloc3", "alloc2"}, 1, 1)}
+ allocs := map[string]string{}
+ allocs["alloc4"] = node1
+ plugin := mock.NewPreemptionPredicatePlugin(nil, allocs, preemptions)
+ plugins.RegisterSchedulerPlugin(plugin)
+ defer plugins.UnregisterSchedulerPlugins()
+
+ alloc, ok := preemptor.TryPreemption()
+ assert.NilError(t, plugin.GetPredicateError())
+ assert.Assert(t, ok, "no victims found")
+ assert.Equal(t, "alloc4", alloc.allocationKey, "wrong alloc")
+ assert.Check(t, !alloc1.IsPreempted(), "alloc1 not preempted")
+ assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted")
+ assert.Check(t, alloc3.IsPreempted(), "alloc3 preempted")
+}
+
+// TestTryPreemption_OnNode_AskResTypesSame_GuaranteedSetOnPreemptorSide Test
try preemption with 2 level queue hierarchy. Since Node doesn't have enough
resources to accomodate, preemption happens because of node resource constraint.
+// Guaranteed set only on preemptor queue path.
+// Ask (Preemptor) resource type matches with one of the victim's resource
types. Still, needs to be preempted because matching resource type has been
configured as guaranteed.
+// Setup:
+// Nodes are Node1. Node is full. Doesn't have space to accomodate the ask.
+// root.parent.parent1.child1. Guaranteed set on parent1, vcores: 2. Ask of
vcores: 2, mem: 200 is waiting for resources.
+// root.parent.parent2.child2. Guaranteed set on parent2, vcores: 1. 6
Allocations (belongs to two diff apps) are running. 3 Allocation's usage is
vcores:1. 3 more allocations usage gpu: 100. Total usage is vcores:3, gpu: 300
+// root.parent.parent2.child3. No usage, no guaranteed set
+// 3 Allocations of each vcores:1 running on root.parent.parent1.child2 could
be preempted to free up resources for ask arrived in root.parent.parent1.child1.
+// but last allocation should not be touched as preempting the same would make
usage goes above the guaranteed set on preemptor or ask queue
root.parent.parent2.child1.
+// All remaining three allocation of each mem: 100 should not be touched at
all as there is no matching resource type between these allocs and ask resource
types.
+func TestTryPreemption_OnNode_AskResTypesSame_GuaranteedSetOnPreemptorSide(t
*testing.T) {
+ t.SkipNow()
+ node := newNode(node1, map[string]resources.Quantity{"vcores": 3,
"gpu": 300, "mem": 200})
+ iterator := getNodeIteratorFn(node)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
nil, nil)
+ assert.NilError(t, err)
+ parentQ1, err := createManagedQueueGuaranteed(parentQ, "parent1", true,
nil, nil)
+ assert.NilError(t, err)
+ parentQ2, err := createManagedQueueGuaranteed(parentQ, "parent2", true,
nil, nil)
+ assert.NilError(t, err)
+
+ childQ1, err := createManagedQueueGuaranteed(parentQ1, "child1", false,
nil, map[string]string{"vcores": "2"})
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ2, "child2", false,
nil, nil)
+ assert.NilError(t, err)
+ _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil,
nil)
+ assert.NilError(t, err)
+
+ app1 := newApplication(appID1, "default", "root.parent.parent2.child2")
+ app1.SetQueue(childQ2)
+ childQ2.applications[appID1] = app1
+ app2 := newApplication(appID2, "default", "root.parent.parent2.child2")
+ app2.SetQueue(childQ2)
+ childQ2.applications[appID2] = app2
+ app3 := newApplication(appID3, "default", "root.parent.parent2.child2")
+ app3.SetQueue(childQ2)
+ childQ2.applications[appID3] = app3
+
+ for i := 5; i < 8; i++ {
+ askN := newAllocationAsk(alloc+strconv.Itoa(i), appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"gpu": 100}))
+ askN.createTime = time.Now().Add(-2 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(askN))
+ allocN := NewAllocation(node1, askN)
+ app1.AddAllocation(allocN)
+ assert.Check(t, node.AddAllocation(allocN), "node alloc1
failed")
+ }
+
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ ask1.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ ask2.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ ask3.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask3))
+ alloc1 := NewAllocation(node1, ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation(node1, ask2)
+ app2.AddAllocation(alloc2)
+ assert.Check(t, node.AddAllocation(alloc2), "node alloc2 failed")
+ alloc3 := NewAllocation(node1, ask3)
+ app3.AddAllocation(alloc3)
+ assert.Check(t, node.AddAllocation(alloc3), "node alloc3 failed")
+
+ for i := 5; i < 8; i++ {
+ assert.NilError(t,
childQ2.IncAllocatedResource(resources.NewResourceFromMap(map[string]resources.Quantity{"gpu":
100}), false))
+ }
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask3.GetAllocatedResource(), false))
+
+ app4 := newApplication("app-4", "default", "root.parent.parent1.child1")
+ app4.SetQueue(childQ1)
+ ask4 := newAllocationAsk("alloc4", "app-4",
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2, "mem":
200}))
+ assert.NilError(t, app4.AddAllocationAsk(ask4))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2})
+ preemptor := NewPreemptor(app4, headRoom, 30*time.Second, ask4,
iterator(), false)
+
+ // register predicate handler
+ preemptions := []mock.Preemption{mock.NewPreemption(true, "alloc4",
node1, []string{"alloc3", "alloc2"}, 1, 1)}
+ plugin := mock.NewPreemptionPredicatePlugin(nil, nil, preemptions)
+ plugins.RegisterSchedulerPlugin(plugin)
+ defer plugins.UnregisterSchedulerPlugins()
+
+ alloc, ok := preemptor.TryPreemption()
+ assert.NilError(t, plugin.GetPredicateError())
+ assert.Assert(t, ok, "no victims found")
+ assert.Equal(t, "alloc4", alloc.allocationKey, "wrong alloc")
+ assert.Check(t, !alloc1.IsPreempted(), "alloc1 not preempted")
+ assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted")
+ assert.Check(t, alloc3.IsPreempted(), "alloc3 preempted")
+}
+
+// TestTryPreemption_AskResTypesSame_GuaranteedSetOnVictimAndPreemptorSides
Test try preemption with 2 level queue hierarchy.
+// Guaranteed set on both victim queue path and preemptor queue path.
+// Ask (Preemptor) resource type matches with one of the victim's resource
types. Still, needs to be preempted because matching resource type has been
configured as guaranteed.
+// Setup:
+// Nodes are Node1. Node has enough space to accommodate the new ask.
+// root.parent.parent1.child1. Guaranteed set on parent1, vcores: 2. Ask of
vcores: 2, mem: 200 is waiting for resources.
+// root.parent.parent2.child2. Guaranteed set on parent2, vcores: 1. 6
Allocations (belongs to two diff apps) are running. 3 Allocation's usage is
vcores:1. 3 more allocations usage gpu: 100. Total usage is vcores:3, gpu: 300
+// root.parent.parent2.child3. No usage, no guaranteed set
+// 2 Allocations of each vcores:1 running on root.parent.parent1.child2 could
be preempted to free up resources for ask arrived in root.parent.parent1.child1.
+// 3rd allocation of vcores:1 should not be touched as preempting the same
would make usage goes below the guaranteed set on root.parent.parent2.child2.
+// All remaining three allocation of each mem: 100 should not be touched at
all as there is no matching resource type between these allocs and ask resource
types.
+func
TestTryPreemption_AskResTypesSame_GuaranteedSetOnVictimAndPreemptorSides(t
*testing.T) {
+ t.SkipNow()
+ node := newNode(node1, map[string]resources.Quantity{"vcores": 5,
"gpu": 700, "mem": 200})
+ iterator := getNodeIteratorFn(node)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
nil, nil)
+ assert.NilError(t, err)
+ parentQ1, err := createManagedQueueGuaranteed(parentQ, "parent1", true,
nil, nil)
+ assert.NilError(t, err)
+ parentQ2, err := createManagedQueueGuaranteed(parentQ, "parent2", true,
nil, nil)
+ assert.NilError(t, err)
+
+ childQ1, err := createManagedQueueGuaranteed(parentQ1, "child1", false,
nil, map[string]string{"vcores": "2"})
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ2, "child2", false,
nil, map[string]string{"vcores": "1"})
+ assert.NilError(t, err)
+ _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil,
nil)
+ assert.NilError(t, err)
+
+ app1 := newApplication(appID1, "default", "root.parent.parent2.child2")
+ app1.SetQueue(childQ2)
+ childQ2.applications[appID1] = app1
+ app2 := newApplication(appID2, "default", "root.parent.parent2.child2")
+ app2.SetQueue(childQ2)
+ childQ2.applications[appID2] = app2
+ app3 := newApplication(appID3, "default", "root.parent.parent2.child2")
+ app3.SetQueue(childQ2)
+ childQ2.applications[appID3] = app3
+
+ for i := 5; i < 8; i++ {
+ askN := newAllocationAsk(alloc+strconv.Itoa(i), appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"gpu": 100}))
+ askN.createTime = time.Now().Add(-2 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(askN))
+ allocN := NewAllocation(node1, askN)
+ app1.AddAllocation(allocN)
+ assert.Check(t, node.AddAllocation(allocN), "node alloc1
failed")
+ }
+
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ ask1.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ ask2.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ ask3.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask3))
+ alloc1 := NewAllocation(node1, ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation(node1, ask2)
+ app2.AddAllocation(alloc2)
+ assert.Check(t, node.AddAllocation(alloc2), "node alloc2 failed")
+ alloc3 := NewAllocation(node1, ask3)
+ app3.AddAllocation(alloc3)
+ assert.Check(t, node.AddAllocation(alloc3), "node alloc3 failed")
+
+ for i := 5; i < 8; i++ {
+ assert.NilError(t,
childQ2.IncAllocatedResource(resources.NewResourceFromMap(map[string]resources.Quantity{"gpu":
100}), false))
+ }
+
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask3.GetAllocatedResource(), false))
+
+ app4 := newApplication("app-4", "default", "root.parent.parent1.child1")
+ app4.SetQueue(childQ1)
+ ask4 := newAllocationAsk("alloc4", "app-4",
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2, "mem":
200}))
+ assert.NilError(t, app4.AddAllocationAsk(ask4))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2})
+ preemptor := NewPreemptor(app4, headRoom, 30*time.Second, ask4,
iterator(), false)
+
+ // register predicate handler
+ preemptions := []mock.Preemption{mock.NewPreemption(true, "alloc4",
node1, []string{"alloc3", "alloc2"}, 1, 1)}
+ allocs := map[string]string{}
+ allocs["alloc4"] = node1
+ plugin := mock.NewPreemptionPredicatePlugin(nil, allocs, preemptions)
+ plugins.RegisterSchedulerPlugin(plugin)
+ defer plugins.UnregisterSchedulerPlugins()
+
+ alloc, ok := preemptor.TryPreemption()
+ assert.NilError(t, plugin.GetPredicateError())
+ assert.Assert(t, ok, "no victims found")
+ assert.Equal(t, "alloc4", alloc.allocationKey, "wrong alloc")
+ assert.Check(t, !alloc1.IsPreempted(), "alloc1 not preempted")
+ assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted")
+ assert.Check(t, alloc3.IsPreempted(), "alloc3 preempted")
+}
+
+//
TestTryPreemption_OnNode_AskResTypesSame_GuaranteedSetOnVictimAndPreemptorSides
Test try preemption with 2 level queue hierarchy. Since Node doesn't have
enough resources to accomodate, preemption happens because of node resource
constraint.
+// Guaranteed set on both victim queue path and preemptor queue path.
+// Ask (Preemptor) resource type matches with one of the victim's resource
types. Still, needs to be preempted because matching resource type has been
configured as guaranteed.
+// Setup:
+// Nodes are Node1. Node is full. Doesn't have enough space to accomodate the
ask.
+// root.parent.parent1.child1. Guaranteed set on parent1, vcores: 2. Ask of
vcores: 2, mem: 200 is waiting for resources.
+// root.parent.parent2.child2. Guaranteed set on parent2, vcores: 1. 6
Allocations (belongs to two diff apps) are running. 3 Allocation's usage is
vcores:1. 3 more allocations usage gpu: 100. Total usage is vcores:3, gpu: 300
+// root.parent.parent2.child3. No usage, no guaranteed set
+// 2 Allocations of each vcores:1 running on root.parent.parent1.child2 could
be preempted to free up resources for ask arrived in root.parent.parent1.child1.
+// 3rd allocation of vcores:1 should not be touched as preempting the same
would make usage goes below the guaranteed set on root.parent.parent2.child2.
+// All remaining three allocation of each mem: 100 should not be touched at
all as there is no matching resource type between these allocs and ask resource
types.
+func
TestTryPreemption_OnNode_AskResTypesSame_GuaranteedSetOnVictimAndPreemptorSides(t
*testing.T) {
+ t.SkipNow()
+ node := newNode(node1, map[string]resources.Quantity{"vcores": 3,
"gpu": 700, "mem": 200})
+ iterator := getNodeIteratorFn(node)
+ rootQ, err := createRootQueue(nil)
+ assert.NilError(t, err)
+ parentQ, err := createManagedQueueGuaranteed(rootQ, "parent", true,
nil, nil)
+ assert.NilError(t, err)
+ parentQ1, err := createManagedQueueGuaranteed(parentQ, "parent1", true,
nil, nil)
+ assert.NilError(t, err)
+ parentQ2, err := createManagedQueueGuaranteed(parentQ, "parent2", true,
nil, nil)
+ assert.NilError(t, err)
+
+ childQ1, err := createManagedQueueGuaranteed(parentQ1, "child1", false,
nil, map[string]string{"vcores": "2"})
+ assert.NilError(t, err)
+ childQ2, err := createManagedQueueGuaranteed(parentQ2, "child2", false,
nil, map[string]string{"vcores": "1"})
+ assert.NilError(t, err)
+ _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil,
nil)
+ assert.NilError(t, err)
+
+ app1 := newApplication(appID1, "default", "root.parent.parent2.child2")
+ app1.SetQueue(childQ2)
+ childQ2.applications[appID1] = app1
+ app2 := newApplication(appID2, "default", "root.parent.parent2.child2")
+ app2.SetQueue(childQ2)
+ childQ2.applications[appID2] = app2
+ app3 := newApplication(appID3, "default", "root.parent.parent2.child2")
+ app3.SetQueue(childQ2)
+ childQ2.applications[appID3] = app3
+
+ for i := 5; i < 8; i++ {
+ askN := newAllocationAsk(alloc+strconv.Itoa(i), appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"gpu": 100}))
+ askN.createTime = time.Now().Add(-2 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(askN))
+ allocN := NewAllocation(node1, askN)
+ app1.AddAllocation(allocN)
+ assert.Check(t, node.AddAllocation(allocN), "node alloc1
failed")
+ }
+
+ ask1 := newAllocationAsk("alloc1", appID1,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ ask1.createTime = time.Now().Add(-1 * time.Minute)
+ assert.NilError(t, app1.AddAllocationAsk(ask1))
+ ask2 := newAllocationAsk("alloc2", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ ask2.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask2))
+ ask3 := newAllocationAsk("alloc3", appID2,
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 1}))
+ ask3.createTime = time.Now()
+ assert.NilError(t, app1.AddAllocationAsk(ask3))
+ alloc1 := NewAllocation(node1, ask1)
+ app1.AddAllocation(alloc1)
+ assert.Check(t, node.AddAllocation(alloc1), "node alloc1 failed")
+ alloc2 := NewAllocation(node1, ask2)
+ app2.AddAllocation(alloc2)
+ assert.Check(t, node.AddAllocation(alloc2), "node alloc2 failed")
+ alloc3 := NewAllocation(node1, ask3)
+ app3.AddAllocation(alloc3)
+ assert.Check(t, node.AddAllocation(alloc3), "node alloc3 failed")
+
+ for i := 5; i < 8; i++ {
+ assert.NilError(t,
childQ2.IncAllocatedResource(resources.NewResourceFromMap(map[string]resources.Quantity{"gpu":
100}), false))
+ }
+
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask1.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask2.GetAllocatedResource(), false))
+ assert.NilError(t,
childQ2.IncAllocatedResource(ask3.GetAllocatedResource(), false))
+
+ app4 := newApplication("app-4", "default", "root.parent.parent1.child1")
+ app4.SetQueue(childQ1)
+ ask4 := newAllocationAsk("alloc4", "app-4",
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2, "mem":
200}))
+ assert.NilError(t, app4.AddAllocationAsk(ask4))
+ headRoom :=
resources.NewResourceFromMap(map[string]resources.Quantity{"vcores": 2})
+ preemptor := NewPreemptor(app4, headRoom, 30*time.Second, ask4,
iterator(), false)
+
+ // register predicate handler
+ preemptions := []mock.Preemption{mock.NewPreemption(true, "alloc4",
node1, []string{"alloc3", "alloc2"}, 1, 1)}
+ plugin := mock.NewPreemptionPredicatePlugin(nil, nil, preemptions)
+ plugins.RegisterSchedulerPlugin(plugin)
+ defer plugins.UnregisterSchedulerPlugins()
+
+ alloc, ok := preemptor.TryPreemption()
+ assert.NilError(t, plugin.GetPredicateError())
+ assert.Assert(t, ok, "no victims found")
+ assert.Equal(t, "alloc4", alloc.allocationKey, "wrong alloc")
+ assert.Check(t, !alloc1.IsPreempted(), "alloc1 not preempted")
+ assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted")
+ assert.Check(t, alloc3.IsPreempted(), "alloc3 preempted")
+}
+
func TestSolutionScoring(t *testing.T) {
singleAlloc := scoreMap(nodeID1, []bool{false}, []bool{true})
singleOriginator := scoreMap(nodeID1, []bool{true}, []bool{true})
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
