This is an automated email from the ASF dual-hosted git repository.
ccondit pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-k8shim.git
The following commit(s) were added to refs/heads/master by this push:
new 16181374 [YUNIKORN-3039] Add support for pod-level resource requests
(#959)
16181374 is described below
commit 16181374599ac508a3b5e53a288099a65252a4d3
Author: Craig Condit <[email protected]>
AuthorDate: Thu Mar 6 10:04:21 2025 -0600
[YUNIKORN-3039] Add support for pod-level resource requests (#959)
Kubernetes v1.32 adds alpha-level support for pod-level resource
specifications. Add support for this feature in YuniKorn to ensure that
our calculations match what Kubernetes uses.
Closes: #959
---
pkg/cmd/schedulerplugin/main.go | 3 ++
pkg/cmd/shim/main.go | 3 ++
pkg/common/resource.go | 34 ++++++++++--
pkg/common/resource_test.go | 66 ++++++++++++++++++++++++
pkg/plugin/predicates/predicate_manager.go | 19 +++++++
pkg/plugin/predicates/predicate_manager_test.go | 8 +++
scripts/{kind-pod-resize.yaml => kind-1.32.yaml} | 1 +
scripts/run-e2e-tests.sh | 8 +--
8 files changed, 133 insertions(+), 9 deletions(-)
diff --git a/pkg/cmd/schedulerplugin/main.go b/pkg/cmd/schedulerplugin/main.go
index 0651a5eb..66a0a7cd 100644
--- a/pkg/cmd/schedulerplugin/main.go
+++ b/pkg/cmd/schedulerplugin/main.go
@@ -24,9 +24,12 @@ import (
"k8s.io/kubernetes/cmd/kube-scheduler/app"
"github.com/apache/yunikorn-k8shim/pkg/plugin"
+ "github.com/apache/yunikorn-k8shim/pkg/plugin/predicates"
)
func main() {
+ predicates.EnableOptionalKubernetesFeatureGates()
+
command := app.NewSchedulerCommand(
app.WithPlugin(plugin.SchedulerPluginName,
plugin.NewSchedulerPlugin))
diff --git a/pkg/cmd/shim/main.go b/pkg/cmd/shim/main.go
index b053194c..45bd7eaf 100644
--- a/pkg/cmd/shim/main.go
+++ b/pkg/cmd/shim/main.go
@@ -27,6 +27,7 @@ import (
"github.com/apache/yunikorn-k8shim/pkg/client"
"github.com/apache/yunikorn-k8shim/pkg/common/constants"
+ "github.com/apache/yunikorn-k8shim/pkg/plugin/predicates"
"github.com/apache/yunikorn-core/pkg/entrypoint"
"github.com/apache/yunikorn-k8shim/pkg/conf"
@@ -37,6 +38,8 @@ import (
func main() {
log.Log(log.Shim).Info(conf.GetBuildInfoString())
+ predicates.EnableOptionalKubernetesFeatureGates()
+
configMaps, err := client.LoadBootstrapConfigMaps()
if err != nil {
log.Log(log.Shim).Fatal("Unable to bootstrap configuration",
zap.Error(err))
diff --git a/pkg/common/resource.go b/pkg/common/resource.go
index f96456a0..36c8c259 100644
--- a/pkg/common/resource.go
+++ b/pkg/common/resource.go
@@ -22,6 +22,7 @@ import (
"go.uber.org/zap"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
+ helpers "k8s.io/component-helpers/resource"
"github.com/apache/yunikorn-k8shim/pkg/log"
siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common"
@@ -68,12 +69,19 @@ func GetPodResource(pod *v1.Pod) (resource *si.Resource) {
podResource = checkInitContainerRequest(pod, podResource)
}
- // K8s pod EnableOverHead from:
- // alpha: v1.16
- // beta: v1.18
- // Enables PodOverhead, for accounting pod overheads which are specific
to a given RuntimeClass
+ // PodLevelResources feature:
+ // alpha: v1.32
+ // beta: v1.33
+ if pod.Spec.Resources != nil && len(pod.Spec.Resources.Requests) > 0 {
+ // pod-level resources, if present, override sum of
container-level resources
+ // only memory and cpu are supported
+ for name, value := range
getPodLevelResource(pod.Spec.Resources.Requests).GetResources() {
+ podResource.Resources[name] = value
+ }
+ }
- // If Overhead is being utilized, add to the total requests for the pod
+ // K8s EnableOverHead feature:
+ // Enables PodOverhead, for accounting pod overheads which are specific
to a given RuntimeClass
if pod.Spec.Overhead != nil {
podOverHeadResource := getResource(pod.Spec.Overhead)
podResource = Add(podResource, podOverHeadResource)
@@ -268,6 +276,22 @@ func getResource(resourceList v1.ResourceList)
*si.Resource {
return resources.Build()
}
+func getPodLevelResource(resourceList v1.ResourceList) *si.Resource {
+ resources := NewResourceBuilder()
+ for name, value := range resourceList {
+ if helpers.IsSupportedPodLevelResource(name) {
+ switch name {
+ case v1.ResourceCPU:
+ vcore := value.MilliValue()
+ resources.AddResource(siCommon.CPU, vcore)
+ default:
+ resources.AddResource(string(name),
value.Value())
+ }
+ }
+ }
+ return resources.Build()
+}
+
func Equals(left *si.Resource, right *si.Resource) bool {
if left == right {
return true
diff --git a/pkg/common/resource_test.go b/pkg/common/resource_test.go
index da5f5723..2d1d0197 100644
--- a/pkg/common/resource_test.go
+++ b/pkg/common/resource_test.go
@@ -27,6 +27,7 @@ import (
apis "k8s.io/apimachinery/pkg/apis/meta/v1"
k8res "k8s.io/component-helpers/resource"
+ "github.com/apache/yunikorn-k8shim/pkg/plugin/predicates"
siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common"
"github.com/apache/yunikorn-scheduler-interface/lib/go/si"
)
@@ -373,8 +374,73 @@ func siResourceFromList(list v1.ResourceList) *si.Resource
{
return builder.Build()
}
+func TestGetPodResourcesWithPodLevelRequests(t *testing.T) {
+ // ensure required K8s feature gates are enabled
+ predicates.EnableOptionalKubernetesFeatureGates()
+
+ containers := make([]v1.Container, 0)
+
+ // container 01
+ c1Resources := make(map[v1.ResourceName]resource.Quantity)
+ c1Resources[v1.ResourceMemory] = resource.MustParse("500M")
+ c1Resources[v1.ResourceCPU] = resource.MustParse("1")
+ c1Resources["nvidia.com/gpu"] = resource.MustParse("1")
+ containers = append(containers, v1.Container{
+ Name: "container-01",
+ Resources: v1.ResourceRequirements{
+ Requests: c1Resources,
+ },
+ })
+
+ // container 02
+ c2Resources := make(map[v1.ResourceName]resource.Quantity)
+ c2Resources[v1.ResourceMemory] = resource.MustParse("1024M")
+ c2Resources[v1.ResourceCPU] = resource.MustParse("2")
+ c2Resources["nvidia.com/gpu"] = resource.MustParse("4")
+ containers = append(containers, v1.Container{
+ Name: "container-02",
+ Resources: v1.ResourceRequirements{
+ Requests: c2Resources,
+ },
+ })
+
+ // pod
+ pod := &v1.Pod{
+ TypeMeta: apis.TypeMeta{
+ Kind: "Pod",
+ APIVersion: "v1",
+ },
+ ObjectMeta: apis.ObjectMeta{
+ Name: "pod-resource-test-00001",
+ UID: "UID-00001",
+ },
+ Spec: v1.PodSpec{
+ Resources: &v1.ResourceRequirements{
+ Requests: map[v1.ResourceName]resource.Quantity{
+ v1.ResourceMemory:
resource.MustParse("128M"),
+ v1.ResourceCPU:
resource.MustParse("5"),
+ "invalid":
resource.MustParse("1"),
+ },
+ },
+ Containers: containers,
+ },
+ }
+
+ // verify cpu and memory overrides
+ res := GetPodResource(pod)
+ assert.Equal(t, res.Resources[siCommon.Memory].GetValue(),
int64(128*1000*1000))
+ assert.Equal(t, res.Resources[siCommon.CPU].GetValue(), int64(5000))
+ assert.Equal(t, res.Resources["nvidia.com/gpu"].GetValue(), int64(5))
+ assert.Equal(t, res.Resources["pods"].GetValue(), int64(1))
+ _, invalidOk := res.Resources["invalid"]
+ assert.Assert(t, !invalidOk, "invalid should not be present")
+}
+
//nolint:funlen
func TestGetPodResourcesWithInPlacePodVerticalScaling(t *testing.T) {
+ // ensure required K8s feature gates are enabled
+ predicates.EnableOptionalKubernetesFeatureGates()
+
containers := make([]v1.Container, 0)
// container 01
diff --git a/pkg/plugin/predicates/predicate_manager.go
b/pkg/plugin/predicates/predicate_manager.go
index 5e966df6..cff56cd6 100644
--- a/pkg/plugin/predicates/predicate_manager.go
+++ b/pkg/plugin/predicates/predicate_manager.go
@@ -27,9 +27,11 @@ import (
"go.uber.org/zap"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apiserver/pkg/util/feature"
"k8s.io/component-base/config/v1alpha1"
"k8s.io/klog/v2"
schedConfig "k8s.io/kube-scheduler/config/v1"
+ "k8s.io/kubernetes/pkg/features"
apiConfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config/scheme"
"k8s.io/kubernetes/pkg/scheduler/framework"
@@ -272,6 +274,23 @@ func (p *predicateManagerImpl) runFilterPlugin(ctx
context.Context, pl framework
return pl.Filter(ctx, state, pod, nodeInfo)
}
+// EnableOptionalKubernetesFeatureGates ensures that any optional Kubernetes
feature gates that YuniKorn supports are
+// enabled. Currently, as of Kubernetes 1.32, this includes PodLevelResources
and InPlacePodVerticalScaling. These are
+// both safe to enable as part of our default configuration, as they also
require the appropriate feature gates to be
+// enabled at the API server to be functional.
+// This needs to be called before any Kubernetes-specific code is initialized
(ideally top of main()) and in any unit
+// tests which require this functionality.
+func EnableOptionalKubernetesFeatureGates() {
+ log.Log(log.ShimPredicates).Debug("Enabling PodLevelResources feature
gate")
+ if err := feature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=true",
features.PodLevelResources)); err != nil {
+ log.Log(log.ShimPredicates).Fatal("Unable to set
PodLevelResources feature gate", zap.Error(err))
+ }
+ log.Log(log.ShimPredicates).Debug("Enabling InPlacePodVerticalScaling
feature gate")
+ if err := feature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=true",
features.InPlacePodVerticalScaling)); err != nil {
+ log.Log(log.ShimPredicates).Fatal("Unable to set
PodLevelResources feature gate", zap.Error(err))
+ }
+}
+
func NewPredicateManager(handle framework.Handle) PredicateManager {
/*
Default K8S plugins as of 1.32 that implement PreFilter:
diff --git a/pkg/plugin/predicates/predicate_manager_test.go
b/pkg/plugin/predicates/predicate_manager_test.go
index b1104cff..7f2d735c 100644
--- a/pkg/plugin/predicates/predicate_manager_test.go
+++ b/pkg/plugin/predicates/predicate_manager_test.go
@@ -29,10 +29,12 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
+ "k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler/framework"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity"
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity"
@@ -1107,6 +1109,12 @@ func newPodWithPort(hostPorts ...int) *v1.Pod {
}
}
+func TestEnableOptionalKubernetesFeatureGates(t *testing.T) {
+ EnableOptionalKubernetesFeatureGates()
+ assert.Assert(t,
feature.DefaultFeatureGate.Enabled(features.PodLevelResources), "pod level
resources not enabled")
+ assert.Assert(t,
feature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling),
"in-place pod vertical scaling not enabled")
+}
+
func TestRunGeneralPredicates(t *testing.T) {
clientSet := clientSet()
informerFactory := informerFactory(clientSet)
diff --git a/scripts/kind-pod-resize.yaml b/scripts/kind-1.32.yaml
similarity index 97%
rename from scripts/kind-pod-resize.yaml
rename to scripts/kind-1.32.yaml
index ae514fab..581885da 100644
--- a/scripts/kind-pod-resize.yaml
+++ b/scripts/kind-1.32.yaml
@@ -19,6 +19,7 @@ kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
featureGates:
"InPlacePodVerticalScaling": true
+ "PodLevelResources": true
nodes:
- role: control-plane
- role: worker
diff --git a/scripts/run-e2e-tests.sh b/scripts/run-e2e-tests.sh
index 2c82b4c5..0042158c 100755
--- a/scripts/run-e2e-tests.sh
+++ b/scripts/run-e2e-tests.sh
@@ -47,12 +47,12 @@ function verlt() {
function update_kind_config() {
# use a different kind config for different cluster versions
version=$(echo "$1" | sed 's/.*://' | sed 's/^v//')
- if verlt "${version}" "1.27"; then
- # 1.26 or earlier
+ if verlt "${version}" "1.32"; then
+ # 1.31 or earlier
KIND_CONFIG=./scripts/kind.yaml
else
- # 1.27 or later; enable InPlacePodVerticalScaling feature flag
- KIND_CONFIG=./scripts/kind-pod-resize.yaml
+ # 1.32 or later; enable InPlacePodVerticalScaling and PodLevelResources
feature flags
+ KIND_CONFIG=./scripts/kind-1.32.yaml
fi
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
