FrankYang0529 commented on code in PR #634: URL: https://github.com/apache/yunikorn-k8shim/pull/634#discussion_r1263259803
########## test/e2e/user_group_limit/user_group_limit_test.go: ########## @@ -0,0 +1,174 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package user_group_limit_test + +import ( + "fmt" + + "github.com/apache/yunikorn-core/pkg/common/configs" + "github.com/apache/yunikorn-core/pkg/common/resources" + siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + + "github.com/apache/yunikorn-k8shim/pkg/common/constants" + tests "github.com/apache/yunikorn-k8shim/test/e2e" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s" + "github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn" +) + +const sleepPodMem = 100 + +var kClient k8s.KubeCtl +var restClient yunikorn.RClient +var ns *v1.Namespace +var dev = "dev" + common.RandSeq(5) +var oldConfigMap = new(v1.ConfigMap) +var annotation = "ann-" + common.RandSeq(10) + +var _ = ginkgo.BeforeSuite(func() { + // Initializing kubectl client + kClient = k8s.KubeCtl{} + Ω(kClient.SetClient()).To(gomega.BeNil()) + // Initializing rest client + restClient = yunikorn.RClient{} + Ω(restClient).NotTo(gomega.BeNil()) + + yunikorn.EnsureYuniKornConfigsPresent() + + ginkgo.By("Port-forward the scheduler pod") + var err = kClient.PortForwardYkSchedulerPod() + Ω(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("create development namespace") + ns, err = kClient.CreateNamespace(dev, nil) + gomega.Ω(err).NotTo(gomega.HaveOccurred()) + gomega.Ω(ns.Status.Phase).To(gomega.Equal(v1.NamespaceActive)) +}) + +var _ = ginkgo.AfterSuite(func() { + ginkgo.By("Check Yunikorn's health") + checks, err := yunikorn.GetFailedHealthChecks() + Ω(err).NotTo(gomega.HaveOccurred()) + Ω(checks).To(gomega.Equal(""), checks) + + testDescription := ginkgo.CurrentSpecReport() + if testDescription.Failed() { + tests.LogTestClusterInfoWrapper(testDescription.FailureMessage(), []string{ns.Name}) + tests.LogYunikornContainer(testDescription.FailureMessage()) + } + ginkgo.By("Tearing down namespace: " + ns.Name) + err = kClient.TearDownNamespace(ns.Name) + Ω(err).NotTo(gomega.HaveOccurred()) +}) + +var _ = ginkgo.Describe("UserGroupLimit", func() { + ginkgo.It("Verify_maxapplications", func() { + // update config + ginkgo.By("Update root.sandbox1 with 1 maxapplications and 1Gi max memory") + annotation = "ann-" + common.RandSeq(10) + yunikorn.UpdateCustomConfigMapWrapper(oldConfigMap, "", annotation, func(sc *configs.SchedulerConfig) error { + // remove placement rules so we can control queue + sc.Partitions[0].PlacementRules = nil + + var err error + if err = common.AddQueue(sc, "default", "root", configs.QueueConfig{ + Name: "sandbox1", + Limits: []configs.Limit{ + { + Limit: "Test entry", + Users: []string{"kubernetes-admin"}, + MaxApplications: 1, + MaxResources: map[string]string{ + siCommon.Memory: "1Gi", + }, + }, + }, + }); err != nil { + return err + } + return nil + }) + + // Deploy sleepPod1 + sleepPodObjs := createSleepPodObjs(2, map[string]string{"queue": "root.sandbox1"}) + sleepPodObj1 := sleepPodObjs[0] + ginkgo.By("Deploy the sleep pod " + sleepPodObj1.Name + " to the development namespace") + sleepPod1, podErr := kClient.CreatePod(sleepPodObj1, dev) + gomega.Ω(podErr).NotTo(gomega.HaveOccurred()) + sleepPod1Application := sleepPod1.Labels[constants.LabelApplicationID] + + // Wait for sleepPod1 to move to running state + podErr = kClient.WaitForPodBySelectorRunning(dev, + fmt.Sprintf("app=%s", sleepPod1.ObjectMeta.Labels["app"]), + 60) + gomega.Ω(podErr).NotTo(gomega.HaveOccurred()) + + // Deploy sleepPod2 + sleepPodObj2 := sleepPodObjs[1] + ginkgo.By("Deploy the sleep pod " + sleepPodObj2.Name + " to the development namespace") + sleepPod2, podErr := kClient.CreatePod(sleepPodObj2, dev) + gomega.Ω(podErr).NotTo(gomega.HaveOccurred()) + sleepPod2Application := sleepPod2.Labels[constants.LabelApplicationID] + + userUsageDAOInfo, err := restClient.GetUserUsage(constants.DefaultPartition, "kubernetes-admin") + Ω(err).NotTo(gomega.HaveOccurred()) + Ω(userUsageDAOInfo).NotTo(gomega.BeNil()) + Ω(len(userUsageDAOInfo.Queues.Children)).To(gomega.Equal(1)) + + sandboxResourceUsageDAO := userUsageDAOInfo.Queues.Children[0] + Ω(sandboxResourceUsageDAO.QueuePath).To(gomega.Equal("root.sandbox1")) + Ω(sandboxResourceUsageDAO.ResourceUsage).NotTo(gomega.BeNil()) + Ω(sandboxResourceUsageDAO.ResourceUsage.Resources["pods"]).To(gomega.Equal(resources.Quantity(1))) + Ω(len(sandboxResourceUsageDAO.RunningApplications)).To(gomega.Equal(1)) + Ω(sandboxResourceUsageDAO.RunningApplications[0]).To(gomega.Equal(sleepPod1Application)) + Ω(sandboxResourceUsageDAO.RunningApplications[0]).NotTo(gomega.Equal(sleepPod2Application)) + + // // sleepPod2 can't be scheduled because application count is more than maxapplications + // ginkgo.By("The sleep pod " + sleepPod2.Name + " can't be scheduled") + // err = kClient.WaitForPodUnschedulable(sleepPod2, 60*time.Second) + // gomega.Ω(err).NotTo(gomega.HaveOccurred()) Review Comment: Not sure whether I understand User Based Quota Enforcement correctly. If the `maxapplications` is 1, I expect a user can't create more than one application in a queue. However, the second application can be scheduled, but the related resource usage is not tracked in the user tracker. From the source code, it looks like we don't handle the result from [IncreaseTrackedResource](https://github.com/apache/yunikorn-core/blob/0ffd1392c329b185bdd8e51665d48e0502807892/pkg/scheduler/objects/application.go#L1656). @wilfred-s, may you give me some feedback about this? Thank you! -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
