This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-core.git
The following commit(s) were added to refs/heads/master by this push:
new 6808a74d [YUNIKORN-2033] Add MaxApplications in Template (#697)
6808a74d is described below
commit 6808a74d877899c9d11f3e83c8869edf8f21bc6e
Author: brandboat <[email protected]>
AuthorDate: Fri Nov 17 19:35:37 2023 +0100
[YUNIKORN-2033] Add MaxApplications in Template (#697)
Closes: #697
Signed-off-by: Peter Bacsko <[email protected]>
---
pkg/scheduler/objects/queue.go | 1 +
pkg/scheduler/objects/queue_test.go | 18 ++++++++---
pkg/scheduler/objects/template/template.go | 41 +++++++++++++++++--------
pkg/scheduler/objects/template/template_test.go | 13 +++++---
pkg/webservice/dao/queue_info.go | 1 +
pkg/webservice/handlers_test.go | 2 ++
6 files changed, 55 insertions(+), 21 deletions(-)
diff --git a/pkg/scheduler/objects/queue.go b/pkg/scheduler/objects/queue.go
index b371f6c6..d2afbcff 100644
--- a/pkg/scheduler/objects/queue.go
+++ b/pkg/scheduler/objects/queue.go
@@ -208,6 +208,7 @@ func newDynamicQueueInternal(name string, leaf bool, parent
*Queue) (*Queue, err
// applyTemplate uses input template to initialize properties, maxResource,
and guaranteedResource
func (sq *Queue) applyTemplate(childTemplate *template.Template) {
+ sq.maxRunningApps = childTemplate.GetMaxApplications()
sq.properties = childTemplate.GetProperties()
// the resources in template are already checked
sq.guaranteedResource = childTemplate.GetGuaranteedResource()
diff --git a/pkg/scheduler/objects/queue_test.go
b/pkg/scheduler/objects/queue_test.go
index b7bef5d7..2c6fe128 100644
--- a/pkg/scheduler/objects/queue_test.go
+++ b/pkg/scheduler/objects/queue_test.go
@@ -1692,13 +1692,15 @@ func TestGetPartitionQueueDAOInfo(t *testing.T) {
// test template
root.template, err = template.FromConf(&configs.ChildTemplate{
- Properties: getProperties(),
+ MaxApplications: uint64(1),
+ Properties: getProperties(),
Resources: configs.Resources{
Max: getResourceConf(),
Guaranteed: getResourceConf(),
},
})
assert.NilError(t, err)
+ assert.Equal(t, root.template.GetMaxApplications(),
root.GetPartitionQueueDAOInfo().TemplateInfo.MaxApplications)
assert.DeepEqual(t, root.template.GetProperties(),
root.GetPartitionQueueDAOInfo().TemplateInfo.Properties)
assert.DeepEqual(t, root.template.GetMaxResource().DAOMap(),
root.template.GetMaxResource().DAOMap())
assert.DeepEqual(t, root.template.GetGuaranteedResource().DAOMap(),
root.template.GetGuaranteedResource().DAOMap())
@@ -2010,6 +2012,7 @@ func TestSetTemplate(t *testing.T) {
queue, err := createManagedQueueWithProps(nil, "tmp", true, nil, nil)
assert.NilError(t, err, "failed to create basic queue queue: %v", err)
+ maxApplications := uint64(1)
properties := getProperties()
guaranteedResource := getResourceConf()
expectedGuaranteedResource, err :=
resources.NewResourceFromConf(guaranteedResource)
@@ -2019,6 +2022,7 @@ func TestSetTemplate(t *testing.T) {
assert.NilError(t, err, "failed to parse resource: %v", err)
checkTemplate := func(queue *Queue) {
+ assert.Equal(t, queue.template.GetMaxApplications(),
maxApplications)
assert.DeepEqual(t, queue.template.GetProperties(), properties)
assert.DeepEqual(t, queue.template.GetGuaranteedResource(),
expectedGuaranteedResource)
assert.DeepEqual(t, queue.template.GetMaxResource(),
expectedMaxResource)
@@ -2026,7 +2030,8 @@ func TestSetTemplate(t *testing.T) {
// case 0: normal case
err = queue.setTemplate(configs.ChildTemplate{
- Properties: properties,
+ MaxApplications: maxApplications,
+ Properties: properties,
Resources: configs.Resources{
Guaranteed: guaranteedResource,
Max: maxResource,
@@ -2049,7 +2054,8 @@ func TestSetTemplate(t *testing.T) {
func TestApplyTemplate(t *testing.T) {
childTemplate, err := template.FromConf(&configs.ChildTemplate{
- Properties: getProperties(),
+ MaxApplications: uint64(1),
+ Properties: getProperties(),
Resources: configs.Resources{
Max: getResourceConf(),
Guaranteed: getResourceConf(),
@@ -2062,6 +2068,7 @@ func TestApplyTemplate(t *testing.T) {
assert.NilError(t, err, "failed to create basic queue queue: %v", err)
leaf.applyTemplate(childTemplate)
assert.Assert(t, leaf.template == nil)
+ assert.Equal(t, leaf.maxRunningApps, childTemplate.GetMaxApplications())
assert.DeepEqual(t, leaf.properties, childTemplate.GetProperties())
assert.DeepEqual(t, leaf.guaranteedResource,
childTemplate.GetGuaranteedResource())
assert.DeepEqual(t, leaf.maxResource, childTemplate.GetMaxResource())
@@ -2078,6 +2085,7 @@ func TestApplyTemplate(t *testing.T) {
})
assert.NilError(t, err)
leaf2.applyTemplate(zeroTemplate)
+ assert.Assert(t, leaf2.maxRunningApps == 0)
assert.Assert(t, leaf2.template == nil)
assert.Assert(t, leaf2.maxResource == nil)
assert.Assert(t, leaf2.guaranteedResource == nil)
@@ -2284,7 +2292,8 @@ func TestNewDynamicQueue(t *testing.T) {
parent, err := createManagedQueueWithProps(nil, "parent", true, nil,
nil)
assert.NilError(t, err, "failed to create queue: %v", err)
parent.template, err = template.FromConf(&configs.ChildTemplate{
- Properties: getProperties(),
+ MaxApplications: uint64(1),
+ Properties: getProperties(),
Resources: configs.Resources{
Max: getResourceConf(),
Guaranteed: getResourceConf(),
@@ -2296,6 +2305,7 @@ func TestNewDynamicQueue(t *testing.T) {
childLeaf, err := NewDynamicQueue("leaf", true, parent)
assert.NilError(t, err, "failed to create dynamic queue: %v", err)
assert.Assert(t, childLeaf.template == nil)
+ assert.Equal(t, childLeaf.maxRunningApps,
parent.template.GetMaxApplications())
assert.DeepEqual(t, childLeaf.properties,
parent.template.GetProperties())
assert.DeepEqual(t, childLeaf.maxResource,
parent.template.GetMaxResource())
assert.DeepEqual(t, childLeaf.guaranteedResource,
parent.template.GetGuaranteedResource())
diff --git a/pkg/scheduler/objects/template/template.go
b/pkg/scheduler/objects/template/template.go
index 8e90e7a6..2ee601dc 100644
--- a/pkg/scheduler/objects/template/template.go
+++ b/pkg/scheduler/objects/template/template.go
@@ -25,6 +25,7 @@ import (
)
type Template struct {
+ maxApplications uint64
properties map[string]string
maxResource *resources.Resource
guaranteedResource *resources.Resource
@@ -34,17 +35,7 @@ type Template struct {
// It returns error if it fails to parse configs.
// It returns nil value if the configs.ChildTemplate is empty
func FromConf(template *configs.ChildTemplate) (*Template, error) {
- // A non-empty list of empty property values is also empty
- isMapEmpty := func(m map[string]string) bool {
- for k, v := range m {
- if k != "" && v != "" {
- return false
- }
- }
- return true
- }
-
- if template == nil || (isMapEmpty(template.Properties) &&
isMapEmpty(template.Resources.Guaranteed) &&
isMapEmpty(template.Resources.Max)) {
+ if template == nil || isChildTemplateEmpty(template) {
return nil, nil
}
@@ -58,11 +49,29 @@ func FromConf(template *configs.ChildTemplate) (*Template,
error) {
return nil, err
}
- return newTemplate(template.Properties, maxResource,
guaranteedResource), nil
+ return newTemplate(template.MaxApplications, template.Properties,
maxResource, guaranteedResource), nil
+}
+
+func isChildTemplateEmpty(template *configs.ChildTemplate) bool {
+ return template.MaxApplications == 0 &&
+ isMapEmpty(template.Properties) &&
+ isMapEmpty(template.Resources.Guaranteed) &&
+ isMapEmpty(template.Resources.Max)
+}
+
+// A non-empty list of empty property values is also empty
+func isMapEmpty(m map[string]string) bool {
+ for k, v := range m {
+ if k != "" && v != "" {
+ return false
+ }
+ }
+ return true
}
-func newTemplate(properties map[string]string, maxResource
*resources.Resource, guaranteedResource *resources.Resource) *Template {
+func newTemplate(maxApplications uint64, properties map[string]string,
maxResource *resources.Resource, guaranteedResource *resources.Resource)
*Template {
template := &Template{
+ maxApplications: maxApplications,
properties: make(map[string]string),
maxResource: nil,
guaranteedResource: nil,
@@ -84,6 +93,11 @@ func newTemplate(properties map[string]string, maxResource
*resources.Resource,
return template
}
+// GetMaxApplications returns max applications.
+func (t *Template) GetMaxApplications() uint64 {
+ return t.maxApplications
+}
+
// GetProperties returns a copy of properties. An empty map replaces the null
value
func (t *Template) GetProperties() map[string]string {
props := make(map[string]string)
@@ -115,6 +129,7 @@ func (t *Template) GetTemplateInfo() *dao.TemplateInfo {
return nil
}
return &dao.TemplateInfo{
+ MaxApplications: t.GetMaxApplications(),
Properties: t.GetProperties(),
MaxResource: t.maxResource.DAOMap(),
GuaranteedResource: t.guaranteedResource.DAOMap(),
diff --git a/pkg/scheduler/objects/template/template_test.go
b/pkg/scheduler/objects/template/template_test.go
index 4de9244a..138ca737 100644
--- a/pkg/scheduler/objects/template/template_test.go
+++ b/pkg/scheduler/objects/template/template_test.go
@@ -48,13 +48,15 @@ func getResource(t *testing.T) *resources.Resource {
return r
}
-func checkMembers(t *testing.T, template *Template, properties
map[string]string, maxResource *resources.Resource, guaranteedResource
*resources.Resource) {
+func checkMembers(t *testing.T, template *Template, maxApplications uint64,
properties map[string]string, maxResource *resources.Resource,
guaranteedResource *resources.Resource) {
// test inner members
+ assert.Equal(t, template.maxApplications, maxApplications)
assert.DeepEqual(t, template.properties, properties)
assert.DeepEqual(t, template.maxResource, maxResource)
assert.DeepEqual(t, template.guaranteedResource, guaranteedResource)
// test all getters
+ assert.Equal(t, template.GetMaxApplications(), maxApplications)
assert.DeepEqual(t, template.GetProperties(), properties)
assert.DeepEqual(t, template.GetMaxResource(), maxResource)
assert.DeepEqual(t, template.GetGuaranteedResource(),
guaranteedResource)
@@ -64,18 +66,21 @@ func TestNewTemplate(t *testing.T) {
properties := getProperties()
guaranteedResource := getResource(t)
maxResource := getResource(t)
+ maxApplications := uint64(1)
- checkMembers(t, newTemplate(properties, maxResource,
guaranteedResource), properties, maxResource, guaranteedResource)
+ checkMembers(t, newTemplate(maxApplications, properties, maxResource,
guaranteedResource), maxApplications, properties, maxResource,
guaranteedResource)
}
func TestFromConf(t *testing.T) {
+ maxApplications := uint64(1)
properties := getProperties()
guaranteedResourceConf := getResourceConf()
maxResourceConf := getResourceConf()
// case 0: normal case
template, err := FromConf(&configs.ChildTemplate{
- Properties: properties,
+ MaxApplications: maxApplications,
+ Properties: properties,
Resources: configs.Resources{
Max: maxResourceConf,
Guaranteed: guaranteedResourceConf,
@@ -87,7 +92,7 @@ func TestFromConf(t *testing.T) {
assert.NilError(t, err, "failed to parse resource: %v", err)
guaranteedResource, err :=
resources.NewResourceFromConf(guaranteedResourceConf)
assert.NilError(t, err, "failed to parse resource: %v", err)
- checkMembers(t, template, properties, maxResource, guaranteedResource)
+ checkMembers(t, template, maxApplications, properties, maxResource,
guaranteedResource)
// case 1: empty map produces nil template
empty0, err := FromConf(&configs.ChildTemplate{
diff --git a/pkg/webservice/dao/queue_info.go b/pkg/webservice/dao/queue_info.go
index b6f60073..a1e3da21 100644
--- a/pkg/webservice/dao/queue_info.go
+++ b/pkg/webservice/dao/queue_info.go
@@ -18,6 +18,7 @@ limitations under the License.
package dao
type TemplateInfo struct {
+ MaxApplications uint64 `json:"maxApplications,omitempty"`
MaxResource map[string]int64 `json:"maxResource,omitempty"`
GuaranteedResource map[string]int64
`json:"guaranteedResource,omitempty"`
Properties map[string]string `json:"properties,omitempty"`
diff --git a/pkg/webservice/handlers_test.go b/pkg/webservice/handlers_test.go
index cd8960f5..713564d8 100644
--- a/pkg/webservice/handlers_test.go
+++ b/pkg/webservice/handlers_test.go
@@ -154,6 +154,7 @@ partitions:
properties:
application.sort.policy: stateaware
childtemplate:
+ maxapplications: 10
properties:
application.sort.policy: stateaware
resources:
@@ -943,6 +944,7 @@ func TestGetPartitionQueuesHandler(t *testing.T) {
assert.Equal(t, partitionQueuesDao.Children[2].Parent, "root")
assert.Equal(t, len(partitionQueuesDao.Properties), 1)
assert.Equal(t,
partitionQueuesDao.Properties["application.sort.policy"], "stateaware")
+ assert.Equal(t, partitionQueuesDao.TemplateInfo.MaxApplications,
uint64(10))
assert.Equal(t, len(partitionQueuesDao.TemplateInfo.Properties), 1)
assert.Equal(t,
partitionQueuesDao.TemplateInfo.Properties["application.sort.policy"],
"stateaware")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
