craigcondit commented on code in PR #429:
URL: https://github.com/apache/yunikorn-k8shim/pull/429#discussion_r892452816
##########
test/e2e/recovery_and_restart/recovery_and_restart_test.go:
##########
@@ -133,6 +135,130 @@ var _ = ginkgo.Describe("", func() {
gomega.Ω(int64(resMap["vcore"].(float64))).To(gomega.Equal(core))
})
+ ginkgo.It("Verify_SleepJobs_Restart_YK", func() {
+ kClient = k8s.KubeCtl{}
+ Ω(kClient.SetClient()).To(gomega.BeNil())
+ defer restorePortForwarding(&kClient)
+
+ appID1 := normalSleepJobPrefix + "-" + common.RandSeq(5)
+ sleepPodConfig1 := k8s.SleepPodConfig{Name: "normal-sleep-job",
NS: dev, Time: 20, AppID: appID1}
+ pod1, podErr := k8s.InitSleepPod(sleepPodConfig1)
+ Ω(podErr).NotTo(gomega.HaveOccurred())
+ appID2 := normalSleepJobPrefix + "-" + common.RandSeq(5)
+ sleepPodConfig2 := k8s.SleepPodConfig{Name:
"normal-sleep-job-2", NS: dev, Time: 20, AppID: appID2}
+ pod2, podErr2 := k8s.InitSleepPod(sleepPodConfig2)
+ Ω(podErr2).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Submitting two normal sleep jobs")
+ job1 := k8s.InitTestJob(appID1, parallelism, parallelism, pod1)
+ _, createErr := kClient.CreateJob(job1, dev)
+ Ω(createErr).NotTo(gomega.HaveOccurred())
+ job2 := k8s.InitTestJob(appID2, parallelism, parallelism, pod2)
+ _, createErr2 := kClient.CreateJob(job2, dev)
+ Ω(createErr2).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Restart the scheduler pod immediately")
+ restartYunikorn(&kClient)
+
+ ginkgo.By("Listing pods")
+ pods, err := kClient.GetPods(dev)
+ Ω(err).NotTo(gomega.HaveOccurred())
+ fmt.Fprintf(ginkgo.GinkgoWriter, "Total number of pods in
namespace %s: %d\n",
+ dev, len(pods.Items))
+ for _, pod := range pods.Items {
+ fmt.Fprintf(ginkgo.GinkgoWriter, "Pod name:
%-40s\tStatus: %s\n", pod.GetName(), pod.Status.Phase)
+ }
+
+ ginkgo.By("Waiting for sleep pods to finish")
Review Comment:
Do we need to wait for these pods to finish? This adds significant delay to
the test, and we could forcibly kill them off.
##########
test/e2e/recovery_and_restart/recovery_and_restart_test.go:
##########
@@ -133,6 +135,130 @@ var _ = ginkgo.Describe("", func() {
gomega.Ω(int64(resMap["vcore"].(float64))).To(gomega.Equal(core))
})
+ ginkgo.It("Verify_SleepJobs_Restart_YK", func() {
+ kClient = k8s.KubeCtl{}
+ Ω(kClient.SetClient()).To(gomega.BeNil())
+ defer restorePortForwarding(&kClient)
+
+ appID1 := normalSleepJobPrefix + "-" + common.RandSeq(5)
+ sleepPodConfig1 := k8s.SleepPodConfig{Name: "normal-sleep-job",
NS: dev, Time: 20, AppID: appID1}
+ pod1, podErr := k8s.InitSleepPod(sleepPodConfig1)
+ Ω(podErr).NotTo(gomega.HaveOccurred())
+ appID2 := normalSleepJobPrefix + "-" + common.RandSeq(5)
+ sleepPodConfig2 := k8s.SleepPodConfig{Name:
"normal-sleep-job-2", NS: dev, Time: 20, AppID: appID2}
+ pod2, podErr2 := k8s.InitSleepPod(sleepPodConfig2)
+ Ω(podErr2).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Submitting two normal sleep jobs")
+ job1 := k8s.InitTestJob(appID1, parallelism, parallelism, pod1)
+ _, createErr := kClient.CreateJob(job1, dev)
+ Ω(createErr).NotTo(gomega.HaveOccurred())
+ job2 := k8s.InitTestJob(appID2, parallelism, parallelism, pod2)
+ _, createErr2 := kClient.CreateJob(job2, dev)
+ Ω(createErr2).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Restart the scheduler pod immediately")
+ restartYunikorn(&kClient)
+
+ ginkgo.By("Listing pods")
+ pods, err := kClient.GetPods(dev)
+ Ω(err).NotTo(gomega.HaveOccurred())
+ fmt.Fprintf(ginkgo.GinkgoWriter, "Total number of pods in
namespace %s: %d\n",
+ dev, len(pods.Items))
+ for _, pod := range pods.Items {
+ fmt.Fprintf(ginkgo.GinkgoWriter, "Pod name:
%-40s\tStatus: %s\n", pod.GetName(), pod.Status.Phase)
+ }
+
+ ginkgo.By("Waiting for sleep pods to finish")
+ err = kClient.WaitForJobPodsSucceeded(dev, job1.Name,
parallelism, 60*time.Second)
+ Ω(err).NotTo(gomega.HaveOccurred())
+ err = kClient.WaitForJobPodsSucceeded(dev, job2.Name,
parallelism, 60*time.Second)
+ Ω(err).NotTo(gomega.HaveOccurred())
+ })
+
+ ginkgo.It("Verify_GangScheduling_TwoGangs_Restart_YK", func() {
+ kClient = k8s.KubeCtl{}
+ Ω(kClient.SetClient()).To(gomega.BeNil())
+ defer restorePortForwarding(&kClient)
+
+ appID := gangSleepJobPrefix + "-" + common.RandSeq(5)
+ sleepPodConfig := k8s.SleepPodConfig{Name: "gang-sleep-job",
NS: dev, Time: 1, AppID: appID}
+ taskGroups := k8s.InitTaskGroups(sleepPodConfig, taskGroupA,
taskGroupB, parallelism)
+ pod, podErr := k8s.InitSleepPod(sleepPodConfig)
+ Ω(podErr).NotTo(gomega.HaveOccurred())
+ pod = k8s.DecoratePodForGangScheduling(30, "Soft", taskGroupA,
+ taskGroups, pod)
+
+ ginkgo.By("Submitting gang sleep job")
+ job := k8s.InitTestJob(appID, parallelism, parallelism, pod)
+ _, err := kClient.CreateJob(job, dev)
+ Ω(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Waiting job pods to be created")
+ createErr := kClient.WaitForJobPodsCreated(dev, job.Name,
parallelism, 30*time.Second)
+ Ω(createErr).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Waiting for placeholders in task group A (expected
state: Running)")
+ err = kClient.WaitForPlaceholders(dev, taskGroupAprefix,
parallelism, 30*time.Second, v1.PodRunning)
+ Ω(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Waiting for placeholders in task group B (expected
state: Pending)")
+ err = kClient.WaitForPlaceholders(dev, taskGroupBprefix,
parallelism+1, 30*time.Second, v1.PodPending)
+ Ω(err).NotTo(gomega.HaveOccurred())
+
+ ginkgo.By("Restart the scheduler pod")
+ restartYunikorn(&kClient)
+
+ // make sure that Yunikorn's internal state have been properly
restored
+ ginkgo.By("Submit sleep job")
+ sleepJob3AppID := "sleepjob-" + common.RandSeq(5)
+ sleepPod3Configs := k8s.SleepPodConfig{Name: "sleepjob3", NS:
dev, AppID: sleepJob3AppID}
+ sleepPod, podErr2 := k8s.InitSleepPod(sleepPod3Configs)
+ Ω(podErr2).NotTo(gomega.HaveOccurred())
+ sleepRespPod, err = kClient.CreatePod(sleepPod, dev)
+ gomega.Ω(err).NotTo(gomega.HaveOccurred())
+ err = kClient.WaitForPodBySelectorRunning(dev,
+ fmt.Sprintf("applicationId=%s", sleepJob3AppID),
+ 60)
+ gomega.Ω(err).NotTo(gomega.HaveOccurred())
+
+ // After YK is up again, we expect to see the same pods in the
same state
+ ginkgo.By("Verify the number of pods & their status")
+ pods, err2 := kClient.GetPods(dev)
+ Ω(err2).NotTo(gomega.HaveOccurred())
+ var groupAPlaceholderCount int
+ var groupBPlaceholderCount int
+ var jobPodCount int
+ fmt.Fprintf(ginkgo.GinkgoWriter, "Total number of pods in
namespace %s: %d\n",
+ dev, len(pods.Items))
+ for _, pod := range pods.Items {
+ podPhase := pod.Status.Phase
+ podName := pod.GetName()
+ fmt.Fprintf(ginkgo.GinkgoWriter, "Pod name:
%-40s\tStatus: %s\n", podName, podPhase)
+ if strings.HasPrefix(podName, taskGroupAprefix) {
+ groupAPlaceholderCount++
+ Ω(podPhase).To(gomega.Equal(v1.PodRunning))
+ continue
+ }
+ if strings.HasPrefix(podName, taskGroupBprefix) {
+ groupBPlaceholderCount++
+ Ω(podPhase).To(gomega.Equal(v1.PodPending))
+ continue
+ }
+ if strings.HasPrefix(podName, gangSleepJobPrefix) {
+ jobPodCount++
+ Ω(podPhase).To(gomega.Equal(v1.PodPending))
+ continue
+ }
+ }
+ Ω(groupAPlaceholderCount).To(gomega.Equal(parallelism))
+ Ω(groupBPlaceholderCount).To(gomega.Equal(parallelism + 1))
+
+ ginkgo.By("Waiting for sleep pods to finish")
Review Comment:
Same as above. Can we just kill these to save time?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
