This is an automated email from the ASF dual-hosted git repository.
wilfred-s pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-core.git
The following commit(s) were added to refs/heads/master by this push:
new c2867c6d [YUNIKORN-3277] Expose ask backoff and quota preemption in
REST (#1102)
c2867c6d is described below
commit c2867c6d559be645df784fa0a118dad96326f22a
Author: mhasnain <[email protected]>
AuthorDate: Wed Jul 22 01:08:00 2026 +1000
[YUNIKORN-3277] Expose ask backoff and quota preemption in REST (#1102)
Expose ask backoff and quota preemption starttime as part of the queue info.
Refactored the unit test splitting backoff and quota preemption start
time into separate tests.
Closes: #1102
Signed-off-by: Wilfred Spiegelenburg <[email protected]>
---
pkg/scheduler/objects/queue.go | 6 ++++
pkg/scheduler/objects/queue_test.go | 64 +++++++++++++++++++++++++++++++++++++
pkg/webservice/dao/queue_info.go | 62 ++++++++++++++++++-----------------
pkg/webservice/handlers_test.go | 40 +++++++++++++++++++++++
4 files changed, 143 insertions(+), 29 deletions(-)
diff --git a/pkg/scheduler/objects/queue.go b/pkg/scheduler/objects/queue.go
index 8c4e63d6..98dfb654 100644
--- a/pkg/scheduler/objects/queue.go
+++ b/pkg/scheduler/objects/queue.go
@@ -956,6 +956,12 @@ func (sq *Queue) GetPartitionQueueDAOInfo(include bool)
dao.PartitionQueueDAOInf
queueInfo.QuotaPreemptionDelay = sq.quotaPreemptionDelay.String()
queueInfo.IsPriorityFence = sq.priorityPolicy ==
policies.FencePriorityPolicy
queueInfo.PriorityOffset = sq.priorityOffset
+ if !sq.quotaPreemptionStartTime.IsZero() {
+ queueInfo.QuotaPreemptionStartTime =
sq.quotaPreemptionStartTime.UnixNano()
+ }
+ queueInfo.IsQuotaPreemptionRunning = sq.isQuotaPreemptionRunning
+ queueInfo.UnschedAskBackoff = sq.unschedAskBackoff
+ queueInfo.AskBackoffDelay = sq.askBackoffDelay.String()
queueInfo.Properties = make(map[string]string)
for k, v := range sq.properties {
queueInfo.Properties[k] = v
diff --git a/pkg/scheduler/objects/queue_test.go
b/pkg/scheduler/objects/queue_test.go
index 5b3b6ffd..064db72a 100644
--- a/pkg/scheduler/objects/queue_test.go
+++ b/pkg/scheduler/objects/queue_test.go
@@ -39,6 +39,7 @@ import (
"github.com/apache/yunikorn-core/pkg/metrics"
"github.com/apache/yunikorn-core/pkg/scheduler/objects/template"
"github.com/apache/yunikorn-core/pkg/scheduler/policies"
+ "github.com/apache/yunikorn-core/pkg/webservice/dao"
siCommon "github.com/apache/yunikorn-scheduler-interface/lib/go/common"
"github.com/apache/yunikorn-scheduler-interface/lib/go/si"
)
@@ -2015,6 +2016,69 @@ func TestGetPartitionQueueDAOInfo(t *testing.T) {
assert.Equal(t, leafDAO.SortingPolicy, "fifo", "incorrect policy
returned")
}
+func TestGetPartitionQueueDAOInfoQuotaPreemptionFields(t *testing.T) {
+ root, err := createRootQueue(nil)
+ assert.NilError(t, err, "failed to create basic root queue")
+ queue, err := createManagedQueue(root, "quota-queue", false,
map[string]string{"memory": "1000"})
+ assert.NilError(t, err, "failed to create quota queue")
+ startTime := time.Now().Add(time.Hour)
+ queue.quotaPreemptionStartTime = startTime
+ queue.isQuotaPreemptionRunning = true
+ tests := []struct {
+ name string
+ input *Queue
+ target dao.PartitionQueueDAOInfo
+ }{
+ {"default values", root,
dao.PartitionQueueDAOInfo{IsQuotaPreemptionRunning: false}},
+ {"running", queue,
dao.PartitionQueueDAOInfo{QuotaPreemptionStartTime: startTime.UnixNano(),
IsQuotaPreemptionRunning: true}},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := tt.input.GetPartitionQueueDAOInfo(false)
+ assert.Equal(t, got.QuotaPreemptionStartTime,
tt.target.QuotaPreemptionStartTime, "quota preemption start time not exposed
correctly")
+ assert.Equal(t, got.IsQuotaPreemptionRunning,
tt.target.IsQuotaPreemptionRunning, "quota preemption running flag not exposed
correctly")
+ })
+ }
+
+ t.Run("cleared after setQuotaPreemptionState(false)", func(t
*testing.T) {
+ queue.setQuotaPreemptionState(false)
+ got := queue.GetPartitionQueueDAOInfo(false)
+ assert.Equal(t, got.QuotaPreemptionStartTime, int64(0))
+ assert.Equal(t, got.IsQuotaPreemptionRunning, false)
+ })
+}
+
+func TestGetPartitionQueueDAOInfoBackoffFields(t *testing.T) {
+ root, err := createRootQueue(nil)
+ assert.NilError(t, err, "failed to create basic root queue")
+ props := map[string]string{
+ configs.ApplicationUnschedulableAsksBackoffDelay: "123s",
+ configs.ApplicationUnschedulableAsksBackoff: "12",
+ }
+ queue, err := createManagedQueueWithProps(root, "backoff-queue", false,
nil, props)
+ assert.NilError(t, err, "failed to create queue with backoff
properties")
+ tests := []struct {
+ name string
+ input *Queue
+ target dao.PartitionQueueDAOInfo
+ }{
+ {"default values", root, dao.PartitionQueueDAOInfo{
+ UnschedAskBackoff: 0,
+ AskBackoffDelay:
configs.DefaultAskBackOffDelay.String(),
+ }},
+ {"configured backoff properties", queue,
dao.PartitionQueueDAOInfo{UnschedAskBackoff: 12, AskBackoffDelay: (123 *
time.Second).String()}},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := tt.input.GetPartitionQueueDAOInfo(false)
+ assert.Equal(t, got.UnschedAskBackoff,
tt.target.UnschedAskBackoff, "unsched ask backoff not exposed correctly")
+ assert.Equal(t, got.AskBackoffDelay,
tt.target.AskBackoffDelay, "ask backoff delay not exposed correctly")
+ })
+ }
+}
+
func getAllocatingAcceptedApps() map[string]bool {
allocatingAcceptedApps := make(map[string]bool)
allocatingAcceptedApps[appID1] = true
diff --git a/pkg/webservice/dao/queue_info.go b/pkg/webservice/dao/queue_info.go
index 607d366a..d80687b7 100644
--- a/pkg/webservice/dao/queue_info.go
+++ b/pkg/webservice/dao/queue_info.go
@@ -26,33 +26,37 @@ type TemplateInfo struct {
}
type PartitionQueueDAOInfo struct {
- QueueName string `json:"queuename"` // no
omitempty, queue name should not be empty
- Status string `json:"status,omitempty"`
- Partition string `json:"partition"` // no
omitempty, partition name should not be empty
- PendingResource map[string]int64
`json:"pendingResource,omitempty"`
- MaxResource map[string]int64
`json:"maxResource,omitempty"`
- GuaranteedResource map[string]int64
`json:"guaranteedResource,omitempty"`
- AllocatedResource map[string]int64
`json:"allocatedResource,omitempty"`
- PreemptingResource map[string]int64
`json:"preemptingResource,omitempty"`
- HeadRoom map[string]int64
`json:"headroom,omitempty"`
- IsLeaf bool `json:"isLeaf"` // no
omitempty, a false value gives a quick way to understand whether it's leaf.
- IsManaged bool `json:"isManaged"` // no
omitempty, a false value gives a quick way to understand whether it's managed.
- Properties map[string]string
`json:"properties,omitempty"`
- Parent string `json:"parent,omitempty"`
- TemplateInfo *TemplateInfo
`json:"template,omitempty"`
- Children []PartitionQueueDAOInfo
`json:"children,omitempty"`
- ChildNames []string
`json:"childNames,omitempty"`
- AbsUsedCapacity map[string]int64
`json:"absUsedCapacity,omitempty"`
- MaxRunningApps uint64
`json:"maxRunningApps,omitempty"`
- RunningApps uint64
`json:"runningApps,omitempty"`
- CurrentPriority int32 `json:"currentPriority"`
// no omitempty, as the current priority value may be 0, which is a valid
priority level
- AllocatingAcceptedApps []string
`json:"allocatingAcceptedApps,omitempty"`
- SortingPolicy string
`json:"sortingPolicy,omitempty"`
- PrioritySorting bool `json:"prioritySorting"`
// no omitempty, false shows priority sorting status better
- PreemptionEnabled bool
`json:"preemptionEnabled"` // no omitempty, false shows preemption status better
- IsPreemptionFence bool
`json:"isPreemptionFence"` // no omitempty, a false value gives a quick way to
understand whether it's fenced.
- PreemptionDelay string
`json:"preemptionDelay,omitempty"`
- QuotaPreemptionDelay string
`json:"quotaPreemptionDelay,omitempty"`
- IsPriorityFence bool `json:"isPriorityFence"`
// no omitempty, a false value gives a quick way to understand whether it's
fenced.
- PriorityOffset int32
`json:"priorityOffset,omitempty"`
+ QueueName string `json:"queuename"` //
no omitempty, queue name should not be empty
+ Status string
`json:"status,omitempty"`
+ Partition string `json:"partition"` //
no omitempty, partition name should not be empty
+ PendingResource map[string]int64
`json:"pendingResource,omitempty"`
+ MaxResource map[string]int64
`json:"maxResource,omitempty"`
+ GuaranteedResource map[string]int64
`json:"guaranteedResource,omitempty"`
+ AllocatedResource map[string]int64
`json:"allocatedResource,omitempty"`
+ PreemptingResource map[string]int64
`json:"preemptingResource,omitempty"`
+ HeadRoom map[string]int64
`json:"headroom,omitempty"`
+ IsLeaf bool `json:"isLeaf"` //
no omitempty, a false value gives a quick way to understand whether it's leaf.
+ IsManaged bool `json:"isManaged"` //
no omitempty, a false value gives a quick way to understand whether it's
managed.
+ Properties map[string]string
`json:"properties,omitempty"`
+ Parent string
`json:"parent,omitempty"`
+ TemplateInfo *TemplateInfo
`json:"template,omitempty"`
+ Children []PartitionQueueDAOInfo
`json:"children,omitempty"`
+ ChildNames []string
`json:"childNames,omitempty"`
+ AbsUsedCapacity map[string]int64
`json:"absUsedCapacity,omitempty"`
+ MaxRunningApps uint64
`json:"maxRunningApps,omitempty"`
+ RunningApps uint64
`json:"runningApps,omitempty"`
+ CurrentPriority int32
`json:"currentPriority"` // no omitempty, as the current priority value may be
0, which is a valid priority level
+ AllocatingAcceptedApps []string
`json:"allocatingAcceptedApps,omitempty"`
+ SortingPolicy string
`json:"sortingPolicy,omitempty"`
+ PrioritySorting bool
`json:"prioritySorting"` // no omitempty, false shows priority sorting status
better
+ PreemptionEnabled bool
`json:"preemptionEnabled"` // no omitempty, false shows preemption status better
+ IsPreemptionFence bool
`json:"isPreemptionFence"` // no omitempty, a false value gives a quick way to
understand whether it's fenced.
+ PreemptionDelay string
`json:"preemptionDelay,omitempty"`
+ QuotaPreemptionDelay string
`json:"quotaPreemptionDelay,omitempty"`
+ IsPriorityFence bool
`json:"isPriorityFence"` // no omitempty, a false value gives a quick way to
understand whether it's fenced.
+ PriorityOffset int32
`json:"priorityOffset,omitempty"`
+ QuotaPreemptionStartTime int64
`json:"quotaPreemptionStartTime,omitempty"`
+ IsQuotaPreemptionRunning bool
`json:"isQuotaPreemptionRunning"` // no omitempty, false shows quota preemption
status better
+ UnschedAskBackoff uint64
`json:"unschedAskBackoff,omitempty"`
+ AskBackoffDelay string
`json:"askBackoffDelay,omitempty"`
}
diff --git a/pkg/webservice/handlers_test.go b/pkg/webservice/handlers_test.go
index 1ebf7eff..9a635ba8 100644
--- a/pkg/webservice/handlers_test.go
+++ b/pkg/webservice/handlers_test.go
@@ -270,6 +270,19 @@ partitions:
- name: default
`
+const configQueueBackoffProperties = `
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: leaf
+ properties:
+ application.sort.policy: fifo
+ application.unschedasks.backoff: "12"
+ application.unschedasks.backoff.delay: "123s"
+`
+
const rmID = "rm-123"
const policyGroup = "default-policy-group"
const queueName = "root.default"
@@ -1062,6 +1075,14 @@ func assertPartitionQueueDaoInfo(t *testing.T,
partitionQueueDAOInfo *dao.Partit
assert.Equal(t, len(partitionQueueDAOInfo.Properties), 1)
assert.Equal(t,
partitionQueueDAOInfo.Properties[configs.ApplicationSortPolicy],
policies.FifoSortPolicy.String())
assert.DeepEqual(t, partitionQueueDAOInfo.TemplateInfo, templateInfo)
+ assertPartitionQueueDaoInfoBackoffAndQuotaPreemptionFields(t,
partitionQueueDAOInfo, 0, configs.DefaultAskBackOffDelay.String())
+}
+
+func assertPartitionQueueDaoInfoBackoffAndQuotaPreemptionFields(t *testing.T,
partitionQueueDAOInfo *dao.PartitionQueueDAOInfo, unschedAskBackoff uint64,
askBackoffDelay string) {
+ assert.Equal(t, partitionQueueDAOInfo.QuotaPreemptionStartTime,
int64(0), "quota preemption start time should be zero")
+ assert.Equal(t, partitionQueueDAOInfo.IsQuotaPreemptionRunning, false,
"quota preemption should not be running")
+ assert.Equal(t, partitionQueueDAOInfo.UnschedAskBackoff,
unschedAskBackoff, "unsched ask backoff mismatch")
+ assert.Equal(t, partitionQueueDAOInfo.AskBackoffDelay, askBackoffDelay,
"ask backoff delay mismatch")
}
func TestGetPartitionQueueHandler(t *testing.T) {
@@ -1084,6 +1105,7 @@ func TestGetPartitionQueueHandler(t *testing.T) {
assert.Equal(t, len(partitionQueueDao1.Children), 0)
assert.Equal(t, len(partitionQueueDao1.ChildNames), 1)
assert.Equal(t, partitionQueueDao1.ChildNames[0], "root.a.a1")
+ assertPartitionQueueDaoInfoBackoffAndQuotaPreemptionFields(t,
&partitionQueueDao1, 0, configs.DefaultAskBackOffDelay.String())
// test hierarchy queue
var partitionQueueDao2 dao.PartitionQueueDAOInfo
@@ -1098,6 +1120,8 @@ func TestGetPartitionQueueHandler(t *testing.T) {
assert.Equal(t, len(partitionQueueDao2.ChildNames), 1)
assert.Equal(t, partitionQueueDao2.Children[0].QueueName, "root.a.a1")
assert.Equal(t, partitionQueueDao2.ChildNames[0], "root.a.a1")
+ assertPartitionQueueDaoInfoBackoffAndQuotaPreemptionFields(t,
&partitionQueueDao2, 0, configs.DefaultAskBackOffDelay.String())
+ assertPartitionQueueDaoInfoBackoffAndQuotaPreemptionFields(t,
&partitionQueueDao2.Children[0], 0, configs.DefaultAskBackOffDelay.String())
// test partition not exists
req, err = createRequest(t, partitionQueuesHandler+queueA,
map[string]string{"partition": "notexists"})
@@ -1152,6 +1176,22 @@ func TestGetPartitionQueueHandler(t *testing.T) {
assert.Equal(t, errInfo.StatusCode, http.StatusBadRequest)
}
+func TestGetPartitionQueueHandlerBackoffAndQuotaPreemptionFields(t *testing.T)
{
+ setup(t, configQueueBackoffProperties, 1)
+
+ NewWebApp(schedulerContext.Load(), nil)
+
+ req, err := createRequest(t,
"/ws/v1/partition/default/queue/root.leaf", map[string]string{"partition":
"default", "queue": "root.leaf"})
+ assert.NilError(t, err, "HTTP request create failed")
+ resp := &MockResponseWriter{}
+ getPartitionQueue(resp, req)
+ var partitionQueueDao dao.PartitionQueueDAOInfo
+ err = json.Unmarshal(resp.outputBytes, &partitionQueueDao)
+ assert.NilError(t, err, unmarshalError)
+ assert.Equal(t, partitionQueueDao.QueueName, "root.leaf")
+ assertPartitionQueueDaoInfoBackoffAndQuotaPreemptionFields(t,
&partitionQueueDao, 12, (123 * time.Second).String())
+}
+
func TestGetClusterInfo(t *testing.T) {
schedulerContext.Store(&scheduler.ClusterContext{})
// clean up started background routines when done
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
