This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-core.git
The following commit(s) were added to refs/heads/master by this push:
new d2d15ed6 [YUNIKORN-2203] Possible log spew in UGM code (#736)
d2d15ed6 is described below
commit d2d15ed665f3e69764374620a72eb87cce6602de
Author: Peter Bacsko <[email protected]>
AuthorDate: Tue Nov 28 12:32:42 2023 +0100
[YUNIKORN-2203] Possible log spew in UGM code (#736)
Don't log from methods which can be repeatedly called from the scheduling
cycle as it floods the log.
Closes: #736
Signed-off-by: Peter Bacsko <[email protected]>
---
pkg/scheduler/ugm/manager.go | 26 +++++---------------------
pkg/scheduler/ugm/queue_tracker.go | 30 ------------------------------
2 files changed, 5 insertions(+), 51 deletions(-)
diff --git a/pkg/scheduler/ugm/manager.go b/pkg/scheduler/ugm/manager.go
index 0fb12bdc..47f0fb1b 100644
--- a/pkg/scheduler/ugm/manager.go
+++ b/pkg/scheduler/ugm/manager.go
@@ -133,7 +133,7 @@ func (m *Manager) DecreaseTrackedResource(queuePath,
applicationID string, usage
return decreased
}
if removeQT {
- log.Log(log.SchedUGM).Debug("Removing user from manager",
+ log.Log(log.SchedUGM).Info("Removing user from manager",
zap.String("user", user.User))
delete(m.userTrackers, user.User)
}
@@ -159,7 +159,7 @@ func (m *Manager) DecreaseTrackedResource(queuePath,
applicationID string, usage
return decreased
}
if removeQT {
- log.Log(log.SchedUGM).Debug("Removing group from manager",
+ log.Log(log.SchedUGM).Info("Removing group from manager",
zap.String("group", appGroup),
zap.String("queue path", queuePath),
zap.String("application", applicationID),
@@ -219,7 +219,7 @@ func (m *Manager) ensureGroupTrackerForApp(queuePath,
applicationID string, user
if appGroup != common.Empty {
groupTracker = m.GetGroupTracker(appGroup)
if groupTracker == nil {
- log.Log(log.SchedUGM).Debug("Group tracker doesn't
exists. Creating appGroup tracker",
+ log.Log(log.SchedUGM).Info("Group tracker doesn't
exists. Creating appGroup tracker",
zap.String("queue path", queuePath),
zap.String("appGroup", appGroup))
groupTracker = newGroupTracker(appGroup)
@@ -228,7 +228,7 @@ func (m *Manager) ensureGroupTrackerForApp(queuePath,
applicationID string, user
m.Unlock()
}
}
- log.Log(log.SchedUGM).Debug("Group tracker set for user application",
+ log.Log(log.SchedUGM).Info("Group tracker set for user application",
zap.String("appGroup", appGroup),
zap.String("user", user.User),
zap.String("application", applicationID),
@@ -562,7 +562,7 @@ func (m *Manager) getUserTracker(user string) *UserTracker {
if ut, ok := m.userTrackers[user]; ok {
return ut
}
- log.Log(log.SchedUGM).Debug("User tracker doesn't exists. Creating user
tracker.",
+ log.Log(log.SchedUGM).Info("User tracker doesn't exists. Creating user
tracker.",
zap.String("user", user))
userTracker := newUserTracker(user)
m.userTrackers[user] = userTracker
@@ -592,10 +592,6 @@ func (m *Manager) Headroom(queuePath, applicationID
string, user security.UserGr
hierarchy := strings.Split(queuePath, configs.DOT)
userTracker := m.getUserTracker(user.User)
userHeadroom := userTracker.headroom(hierarchy)
- log.Log(log.SchedUGM).Debug("Calculated headroom for user",
- zap.String("user", user.User),
- zap.String("queue path", queuePath),
- zap.Stringer("user headroom", userHeadroom))
// make sure the user has a groupTracker for this application, if not
yet there add it
if !userTracker.hasGroupForApp(applicationID) {
m.ensureGroupTrackerForApp(queuePath, applicationID, user)
@@ -610,10 +606,6 @@ func (m *Manager) Headroom(queuePath, applicationID
string, user security.UserGr
return userHeadroom
}
groupHeadroom := groupTracker.headroom(hierarchy)
- log.Log(log.SchedUGM).Debug("Calculated headroom for group",
- zap.String("group", appGroup),
- zap.String("queue path", queuePath),
- zap.Stringer("group headroom", groupHeadroom))
return resources.ComponentWiseMinPermissive(userHeadroom, groupHeadroom)
}
@@ -622,10 +614,6 @@ func (m *Manager) CanRunApp(queuePath, applicationID
string, user security.UserG
hierarchy := strings.Split(queuePath, configs.DOT)
userTracker := m.getUserTracker(user.User)
userCanRunApp := userTracker.canRunApp(hierarchy, applicationID)
- log.Log(log.SchedUGM).Debug("Check whether user can run app",
- zap.String("user", user.User),
- zap.String("queue path", queuePath),
- zap.Bool("can run app", userCanRunApp))
// make sure the user has a groupTracker for this application, if not
yet there add it
if !userTracker.hasGroupForApp(applicationID) {
m.ensureGroupTrackerForApp(queuePath, applicationID, user)
@@ -640,10 +628,6 @@ func (m *Manager) CanRunApp(queuePath, applicationID
string, user security.UserG
return userCanRunApp
}
groupCanRunApp := groupTracker.canRunApp(hierarchy, applicationID)
- log.Log(log.SchedUGM).Debug("Check whether group can run app",
- zap.String("group", appGroup),
- zap.String("queue path", queuePath),
- zap.Bool("can run app", groupCanRunApp))
return userCanRunApp && groupCanRunApp
}
diff --git a/pkg/scheduler/ugm/queue_tracker.go
b/pkg/scheduler/ugm/queue_tracker.go
index 8a9fa17a..7358cc64 100644
--- a/pkg/scheduler/ugm/queue_tracker.go
+++ b/pkg/scheduler/ugm/queue_tracker.go
@@ -241,11 +241,6 @@ func (qt *QueueTracker) setLimit(hierarchy []string,
maxResource *resources.Reso
}
func (qt *QueueTracker) headroom(hierarchy []string, trackType trackingType)
*resources.Resource {
- log.Log(log.SchedUGM).Debug("Calculating headroom",
- zap.String("queue path", qt.queuePath),
- zap.Strings("hierarchy", hierarchy),
- zap.Int("tracking type", int(trackType)),
- )
// depth first: all the way to the leaf, create if not exists
// more than 1 in the slice means we need to recurse down
var headroom, childHeadroom *resources.Resource
@@ -261,11 +256,6 @@ func (qt *QueueTracker) headroom(hierarchy []string,
trackType trackingType) *re
if !resources.IsZero(qt.maxResources) {
headroom = qt.maxResources.Clone()
headroom.SubOnlyExisting(qt.resourceUsage)
- log.Log(log.SchedUGM).Debug("Calculated headroom",
- zap.String("queue path", qt.queuePath),
- zap.Int("tracking type", int(trackType)),
- zap.Stringer("max resource", qt.maxResources),
- zap.Stringer("headroom", headroom))
} else if resources.IsZero(childHeadroom) {
// If childHeadroom is not nil, it means there is an user or
wildcard limit config in child queue,
// so we don't check wildcard limit config in current queue.
@@ -277,11 +267,6 @@ func (qt *QueueTracker) headroom(hierarchy []string,
trackType trackingType) *re
if config :=
m.getUserWildCardLimitsConfig(qt.queuePath); config != nil {
headroom = config.maxResources.Clone()
headroom.SubOnlyExisting(qt.resourceUsage)
- log.Log(log.SchedUGM).Debug("Calculated
headroom",
- zap.String("queue path", qt.queuePath),
- zap.Int("tracking type",
int(trackType)),
- zap.Stringer("wildcard max resource",
config.maxResources),
- zap.Stringer("headroom", headroom))
}
}
}
@@ -427,11 +412,6 @@ func (qt *QueueTracker)
decreaseTrackedResourceUsageDownwards(hierarchy []string
}
func (qt *QueueTracker) canRunApp(hierarchy []string, applicationID string,
trackType trackingType) bool {
- log.Log(log.SchedUGM).Debug("Checking can run app",
- zap.Int("tracking type", int(trackType)),
- zap.String("queue path", qt.queuePath),
- zap.String("application", applicationID),
- zap.Strings("hierarchy", hierarchy))
// depth first: all the way to the leaf, create if not exists
// more than 1 in the slice means we need to recurse down
childCanRunApp := true
@@ -457,11 +437,6 @@ func (qt *QueueTracker) canRunApp(hierarchy []string,
applicationID string, trac
// apply user/group specific limit settings set if configured,
otherwise use wild card limit settings
if qt.maxRunningApps != 0 && running > int(qt.maxRunningApps) {
- log.Log(log.SchedUGM).Warn("can't run app as allowing new
application to run would exceed configured max applications limit of specific
user/group",
- zap.Int("tracking type", int(trackType)),
- zap.String("queue path", qt.queuePath),
- zap.Int("current running applications",
len(qt.runningApplications)),
- zap.Uint64("max running applications",
qt.maxRunningApps))
return false
}
@@ -474,11 +449,6 @@ func (qt *QueueTracker) canRunApp(hierarchy []string,
applicationID string, trac
config = m.getGroupWildCardLimitsConfig(qt.queuePath)
}
if config != nil && config.maxApplications != 0 && running >
int(config.maxApplications) {
- log.Log(log.SchedUGM).Warn("can't run app as allowing
new application to run would exceed configured max applications limit of
wildcard user/group",
- zap.Int("tracking type", int(trackType)),
- zap.String("queue path", qt.queuePath),
- zap.Int("current running applications",
len(qt.runningApplications)),
- zap.Uint64("max running applications",
config.maxApplications))
return false
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
