pbacsko commented on code in PR #429:
URL: https://github.com/apache/yunikorn-k8shim/pull/429#discussion_r892520332
##########
test/e2e/recovery_and_restart/recovery_and_restart_test.go:
##########
@@ -133,6 +135,130 @@ var _ = ginkgo.Describe("", func() {
gomega.Ω(int64(resMap["vcore"].(float64))).To(gomega.Equal(core))
})
+ ginkgo.It("Verify_SleepJobs_Restart_YK", func() {
+ kClient = k8s.KubeCtl{}
+ Ω(kClient.SetClient()).To(gomega.BeNil())
+ defer restorePortForwarding(&kClient)
+
+ appID1 := normalSleepJobPrefix + "-" + common.RandSeq(5)
+ sleepPodConfig1 := k8s.SleepPodConfig{Name: "normal-sleep-job",
NS: dev, Time: 20, AppID: appID1}
+ pod1, podErr := k8s.InitSleepPod(sleepPodConfig1)
+ Ω(podErr).NotTo(gomega.HaveOccurred())
+ appID2 := normalSleepJobPrefix + "-" + common.RandSeq(5)
+ sleepPodConfig2 := k8s.SleepPodConfig{Name:
"normal-sleep-job-2", NS: dev, Time: 20, AppID: appID2}
+ pod2, podErr2 := k8s.InitSleepPod(sleepPodConfig2)
+ Ω(podErr2).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Submitting two normal sleep jobs")
+ job1 := k8s.InitTestJob(appID1, parallelism, parallelism, pod1)
+ _, createErr := kClient.CreateJob(job1, dev)
+ Ω(createErr).NotTo(gomega.HaveOccurred())
+ job2 := k8s.InitTestJob(appID2, parallelism, parallelism, pod2)
+ _, createErr2 := kClient.CreateJob(job2, dev)
+ Ω(createErr2).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Restart the scheduler pod immediately")
+ restartYunikorn(&kClient)
+
+ ginkgo.By("Listing pods")
+ pods, err := kClient.GetPods(dev)
+ Ω(err).NotTo(gomega.HaveOccurred())
+ fmt.Fprintf(ginkgo.GinkgoWriter, "Total number of pods in
namespace %s: %d\n",
+ dev, len(pods.Items))
+ for _, pod := range pods.Items {
+ fmt.Fprintf(ginkgo.GinkgoWriter, "Pod name:
%-40s\tStatus: %s\n", pod.GetName(), pod.Status.Phase)
+ }
+
+ ginkgo.By("Waiting for sleep pods to finish")
+ err = kClient.WaitForJobPodsSucceeded(dev, job1.Name,
parallelism, 60*time.Second)
+ Ω(err).NotTo(gomega.HaveOccurred())
+ err = kClient.WaitForJobPodsSucceeded(dev, job2.Name,
parallelism, 60*time.Second)
+ Ω(err).NotTo(gomega.HaveOccurred())
+ })
+
+ ginkgo.It("Verify_GangScheduling_TwoGangs_Restart_YK", func() {
+ kClient = k8s.KubeCtl{}
+ Ω(kClient.SetClient()).To(gomega.BeNil())
+ defer restorePortForwarding(&kClient)
+
+ appID := gangSleepJobPrefix + "-" + common.RandSeq(5)
+ sleepPodConfig := k8s.SleepPodConfig{Name: "gang-sleep-job",
NS: dev, Time: 1, AppID: appID}
+ taskGroups := k8s.InitTaskGroups(sleepPodConfig, taskGroupA,
taskGroupB, parallelism)
+ pod, podErr := k8s.InitSleepPod(sleepPodConfig)
+ Ω(podErr).NotTo(gomega.HaveOccurred())
+ pod = k8s.DecoratePodForGangScheduling(30, "Soft", taskGroupA,
+ taskGroups, pod)
+
+ ginkgo.By("Submitting gang sleep job")
+ job := k8s.InitTestJob(appID, parallelism, parallelism, pod)
+ _, err := kClient.CreateJob(job, dev)
+ Ω(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Waiting job pods to be created")
+ createErr := kClient.WaitForJobPodsCreated(dev, job.Name,
parallelism, 30*time.Second)
+ Ω(createErr).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Waiting for placeholders in task group A (expected
state: Running)")
+ err = kClient.WaitForPlaceholders(dev, taskGroupAprefix,
parallelism, 30*time.Second, v1.PodRunning)
+ Ω(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Waiting for placeholders in task group B (expected
state: Pending)")
+ err = kClient.WaitForPlaceholders(dev, taskGroupBprefix,
parallelism+1, 30*time.Second, v1.PodPending)
+ Ω(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Restart the scheduler pod")
+ restartYunikorn(&kClient)
+
+ // make sure that Yunikorn's internal state have been properly
restored
+ ginkgo.By("Submit sleep job")
+ sleepJob3AppID := "sleepjob-" + common.RandSeq(5)
+ sleepPod3Configs := k8s.SleepPodConfig{Name: "sleepjob3", NS:
dev, AppID: sleepJob3AppID}
+ sleepPod, podErr2 := k8s.InitSleepPod(sleepPod3Configs)
+ Ω(podErr2).NotTo(gomega.HaveOccurred())
+ sleepRespPod, err = kClient.CreatePod(sleepPod, dev)
+ gomega.Ω(err).NotTo(gomega.HaveOccurred())
+ err = kClient.WaitForPodBySelectorRunning(dev,
+ fmt.Sprintf("applicationId=%s", sleepJob3AppID),
+ 60)
+ gomega.Ω(err).NotTo(gomega.HaveOccurred())
+
+ // After YK is up again, we expect to see the same pods in the
same state
+ ginkgo.By("Verify the number of pods & their status")
+ pods, err2 := kClient.GetPods(dev)
+ Ω(err2).NotTo(gomega.HaveOccurred())
+ var groupAPlaceholderCount int
+ var groupBPlaceholderCount int
+ var jobPodCount int
+ fmt.Fprintf(ginkgo.GinkgoWriter, "Total number of pods in
namespace %s: %d\n",
+ dev, len(pods.Items))
+ for _, pod := range pods.Items {
+ podPhase := pod.Status.Phase
+ podName := pod.GetName()
+ fmt.Fprintf(ginkgo.GinkgoWriter, "Pod name:
%-40s\tStatus: %s\n", podName, podPhase)
+ if strings.HasPrefix(podName, taskGroupAprefix) {
+ groupAPlaceholderCount++
+ Ω(podPhase).To(gomega.Equal(v1.PodRunning))
+ continue
+ }
+ if strings.HasPrefix(podName, taskGroupBprefix) {
+ groupBPlaceholderCount++
+ Ω(podPhase).To(gomega.Equal(v1.PodPending))
+ continue
+ }
+ if strings.HasPrefix(podName, gangSleepJobPrefix) {
+ jobPodCount++
+ Ω(podPhase).To(gomega.Equal(v1.PodPending))
+ continue
+ }
+ }
+ Ω(groupAPlaceholderCount).To(gomega.Equal(parallelism))
+ Ω(groupBPlaceholderCount).To(gomega.Equal(parallelism + 1))
+
+ ginkgo.By("Waiting for sleep pods to finish")
Review Comment:
In this case, we have to wait the placeholder timeout to make sure that they
indeed terminate and then the transition/replacement occurs properly. Sleep
time here is set to just 1 second, so it's not a big issue.
Adding a short comment to make it clear.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
