FrankYang0529 commented on code in PR #643:
URL: https://github.com/apache/yunikorn-k8shim/pull/643#discussion_r1277076876
##########
test/e2e/preemption/preemption_test.go:
##########
@@ -328,6 +335,223 @@ var _ = ginkgo.Describe("Preemption", func() {
}
})
+ ginkgo.It("Verify_preemption_on_priority_queue", func() {
+ ginkgo.By("A task can only preempt a task with lower or equal
priority")
+ // update config
+ ginkgo.By(fmt.Sprintf("Update root.sandbox1, root.low-priority,
root.high-priority with guaranteed memory %dM", sleepPodMemLimit))
+ annotation = "ann-" + common.RandSeq(10)
+ yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "",
annotation, func(sc *configs.SchedulerConfig) error {
+ // remove placement rules so we can control queue
+ sc.Partitions[0].PlacementRules = nil
+
+ var err error
+ if err = common.AddQueue(sc, "default", "root",
configs.QueueConfig{
+ Name: "high-priority",
+ Resources: configs.Resources{Guaranteed:
map[string]string{"memory": fmt.Sprintf("%dM", sleepPodMemLimit)}},
+ Properties:
map[string]string{"preemption.delay": "1s", "priority.offset": "100"},
+ }); err != nil {
+ return err
+ }
+
+ if err = common.AddQueue(sc, "default", "root",
configs.QueueConfig{
+ Name: "sandbox1",
+ Resources: configs.Resources{Guaranteed:
map[string]string{"memory": fmt.Sprintf("%dM", sleepPodMemLimit)}},
+ Properties:
map[string]string{"preemption.delay": "1s", "priority.offset": "0"},
+ }); err != nil {
+ return err
+ }
+
+ if err = common.AddQueue(sc, "default", "root",
configs.QueueConfig{
+ Name: "low-priority",
+ Resources: configs.Resources{Guaranteed:
map[string]string{"memory": fmt.Sprintf("%dM", sleepPodMemLimit)}},
+ Properties:
map[string]string{"preemption.delay": "1s", "priority.offset": "-100"},
+ }); err != nil {
+ return err
+ }
+ return nil
+ })
+
+ // Define sleepPod
+ sandbox1SleepPodConfigs := createSandbox1SleepPodCofigs(3, 30)
+ sleepPod4Config := k8s.SleepPodConfig{Name: "sleepjob4", NS:
dev, Mem: sleepPodMemLimit, Time: 600, Optedout: true, Labels:
map[string]string{"queue": "root.low-priority"}}
+ sleepPod5Config := k8s.SleepPodConfig{Name: "sleepjob5", NS:
dev, Mem: sleepPodMemLimit, Time: 600, Optedout: true, Labels:
map[string]string{"queue": "root.high-priority"}}
+
+ for _, config := range sandbox1SleepPodConfigs {
+ ginkgo.By("Deploy the sleep pod " + config.Name + " to
the development namespace")
+ sleepObj, podErr := k8s.InitSleepPod(config)
+ Ω(podErr).NotTo(gomega.HaveOccurred())
+ sleepRespPod, podErr := kClient.CreatePod(sleepObj, dev)
+ gomega.Ω(podErr).NotTo(gomega.HaveOccurred())
+
+ // Wait for pod to move to running state
+ podErr = kClient.WaitForPodBySelectorRunning(dev,
+ fmt.Sprintf("app=%s",
sleepRespPod.ObjectMeta.Labels["app"]),
+ 60)
+ gomega.Ω(podErr).NotTo(gomega.HaveOccurred())
+ }
+
+ // Deploy sleepjob4 pod in root.low-priority
+ ginkgo.By("Deploy the sleep pod " + sleepPod4Config.Name + " to
the development namespace")
+ sleepObj, podErr := k8s.InitSleepPod(sleepPod4Config)
+ Ω(podErr).NotTo(gomega.HaveOccurred())
+ sleepRespPod4, err := kClient.CreatePod(sleepObj, dev)
+ gomega.Ω(err).NotTo(gomega.HaveOccurred())
+
+ // Deploy sleepjob5 pod in root.high-priority
+ ginkgo.By("Deploy the sleep pod " + sleepPod5Config.Name + " to
the development namespace")
+ sleepObj, podErr = k8s.InitSleepPod(sleepPod5Config)
+ Ω(podErr).NotTo(gomega.HaveOccurred())
+ sleepRespPod5, err := kClient.CreatePod(sleepObj, dev)
+ gomega.Ω(err).NotTo(gomega.HaveOccurred())
+
+ // sleepjob4 pod can't be scheduled before pods in
root.sandbox1 are succeeded
+ ginkgo.By("The sleep pod " + sleepPod4Config.Name + " can't be
scheduled")
+ err = kClient.WaitForPodUnschedulable(sleepRespPod4,
30*time.Second)
+ gomega.Ω(err).NotTo(gomega.HaveOccurred())
+
+ // sleepjob5 pod can be scheduled before pods in root.sandbox1
are succeeded
+ ginkgo.By("The sleep pod " + sleepPod5Config.Name + " can be
scheduled")
+ err = kClient.WaitForPodScheduled(ns.Name, sleepRespPod5.Name,
30*time.Second)
+ gomega.Ω(err).NotTo(gomega.HaveOccurred())
+
+ // assert one of the pods in root.sandbox1 is preempted
+ ginkgo.By("One of the pods in root.sanbox1 is preempted")
+ sandbox1RunningPodsCnt := 0
+ pods, err := kClient.ListPodsByLabelSelector(dev,
"queue=root.sandbox1")
+ gomega.Ω(err).NotTo(gomega.HaveOccurred())
+ for _, pod := range pods.Items {
+ if pod.DeletionTimestamp != nil {
+ continue
+ }
+ if pod.Status.Phase == v1.PodRunning {
+ sandbox1RunningPodsCnt++
+ }
+ }
+ Ω(sandbox1RunningPodsCnt).To(gomega.Equal(2), "One of the pods
in root.sandbox1 should be preempted")
+ })
+
+ ginkgo.It("Verify_allow_preemption_tag", func() {
+ ginkgo.By("The value of 'false' for the allow preemption
annotation on the PriorityClass moves the Pod to the back of the preemption
list")
+ // update config
+ ginkgo.By(fmt.Sprintf("Update root.sandbox1, root.sandbox2 and
root.sandbox3 with guaranteed memory %dM", sleepPodMemLimit2))
+ annotation = "ann-" + common.RandSeq(10)
+ yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "",
annotation, func(sc *configs.SchedulerConfig) error {
+ // remove placement rules so we can control queue
+ sc.Partitions[0].PlacementRules = nil
+
+ var err error
+ if err = common.AddQueue(sc, "default", "root",
configs.QueueConfig{
+ Name: "sandbox1",
+ Resources: configs.Resources{Guaranteed:
map[string]string{"memory": fmt.Sprintf("%dM", sleepPodMemLimit2)}},
+ Properties:
map[string]string{"preemption.delay": "1s"},
+ }); err != nil {
+ return err
+ }
+
+ if err = common.AddQueue(sc, "default", "root",
configs.QueueConfig{
+ Name: "sandbox2",
+ Resources: configs.Resources{Guaranteed:
map[string]string{"memory": fmt.Sprintf("%dM", sleepPodMemLimit2)}},
+ Properties:
map[string]string{"preemption.delay": "1s"},
+ }); err != nil {
+ return err
+ }
+
+ if err = common.AddQueue(sc, "default", "root",
configs.QueueConfig{
+ Name: "sandbox3",
+ Resources: configs.Resources{Guaranteed:
map[string]string{"memory": fmt.Sprintf("%dM", sleepPodMemLimit2)}},
+ Properties:
map[string]string{"preemption.delay": "1s"},
+ }); err != nil {
+ return err
+ }
+ return nil
+ })
+
+ // Define PriorityClass
+ var preemptAllowPriorityClass = schedulingv1.PriorityClass{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "allow-preemption",
+ Annotations:
map[string]string{constants.AnnotationAllowPreemption: constants.True},
+ },
+ }
+ var preemptNotAllowPriorityClass = schedulingv1.PriorityClass{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "preemption-not-allow",
+ Annotations:
map[string]string{constants.AnnotationAllowPreemption: constants.False},
+ },
+ }
+
+ // Create PriorityClass
+ ginkgo.By(fmt.Sprintf("Creating priority class %s",
preemptAllowPriorityClass.Name))
+ _, err :=
kClient.CreatePriorityClass(&preemptAllowPriorityClass)
+ gomega.Ω(err).ShouldNot(HaveOccurred())
+ ginkgo.By(fmt.Sprintf("Creating priority class %s",
preemptNotAllowPriorityClass.Name))
+ _, err =
kClient.CreatePriorityClass(&preemptNotAllowPriorityClass)
+ gomega.Ω(err).ShouldNot(HaveOccurred())
+
+ // Define sleepPod
+ sleepPod1Config := k8s.SleepPodConfig{Name: "sleepjob1", NS:
dev, Mem: sleepPodMemLimit2, Time: 60, Optedout: true, Labels:
map[string]string{"queue": "root.sandbox1"}}
+ sleepPod2Config := k8s.SleepPodConfig{Name: "sleepjob2", NS:
dev, Mem: sleepPodMemLimit2, Time: 60, Optedout: true, Labels:
map[string]string{"queue": "root.sandbox1"}}
+ sleepPod3Config := k8s.SleepPodConfig{Name: "sleepjob3", NS:
dev, Mem: sleepPodMemLimit2, Time: 60, Optedout: false, Labels:
map[string]string{"queue": "root.sandbox2"}}
+ sleepPod4Config := k8s.SleepPodConfig{Name: "sleepjob4", NS:
dev, Mem: sleepPodMemLimit2, Time: 60, Optedout: false, Labels:
map[string]string{"queue": "root.sandbox2"}}
Review Comment:
The `Optedout` config didn't take effect. I fix it in
https://github.com/apache/yunikorn-k8shim/pull/646. If we want to test
`yunikorn.apache.org/allow-preemption` annotation key in PriorityClass, we may
need to remove Pod annotations and only keep PriorityClassName, because we
check annotations from Pod first and then PriorityClass.
https://github.com/apache/yunikorn-k8shim/blob/ac8df7992c94b9b5c36b28452e2de1b5a050f373/pkg/cache/task.go#L251-L261
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
