[hadoop] branch trunk updated: YARN-1115: Provide optional means for a scheduler to check real user ACLs. Contributed by Eric Payne (epayne)
This is an automated email from the ASF dual-hosted git repository.
ahussein pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d286994 YARN-1115: Provide optional means for a scheduler to check
real user ACLs. Contributed by Eric Payne (epayne)
d286994 is described below
commit d2869940094d330434f3e82d16b1cad3c6023437
Author: Ahmed Hussein
AuthorDate: Wed Oct 20 22:18:36 2021 +
YARN-1115: Provide optional means for a scheduler to check real user ACLs.
Contributed by Eric Payne (epayne)
---
.../server/resourcemanager/ClientRMService.java| 6 +-
.../yarn/server/resourcemanager/RMAppManager.java | 32 -
.../resourcemanager/recovery/RMStateStore.java | 5 +-
.../recovery/records/ApplicationStateData.java | 45 ++-
.../impl/pb/ApplicationStateDataPBImpl.java| 17 ++-
.../yarn/server/resourcemanager/rmapp/RMApp.java | 2 +
.../server/resourcemanager/rmapp/RMAppImpl.java| 25 +++-
.../scheduler/capacity/CapacityScheduler.java | 6 +-
.../yarn_server_resourcemanager_recovery.proto | 1 +
.../server/resourcemanager/AppManagerTestBase.java | 3 +-
.../yarn/server/resourcemanager/RMHATestBase.java | 4 +-
.../applicationsmanager/MockAsm.java | 4 +
.../server/resourcemanager/rmapp/MockRMApp.java| 5 +
.../scheduler/capacity/TestLeafQueue.java | 144 +
.../src/site/markdown/CapacityScheduler.md | 4 +-
15 files changed, 283 insertions(+), 20 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index dbadc3c..04efc88 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -600,10 +600,12 @@ public class ClientRMService extends AbstractService
implements
// checked here, those that are dependent on RM configuration are validated
// in RMAppManager.
+UserGroupInformation userUgi = null;
String user = null;
try {
// Safety
- user = UserGroupInformation.getCurrentUser().getShortUserName();
+ userUgi = UserGroupInformation.getCurrentUser();
+ user = userUgi.getShortUserName();
} catch (IOException ie) {
LOG.warn("Unable to get the current user.", ie);
RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
@@ -694,7 +696,7 @@ public class ClientRMService extends AbstractService
implements
try {
// call RMAppManager to submit application directly
rmAppManager.submitApplication(submissionContext,
- System.currentTimeMillis(), user);
+ System.currentTimeMillis(), userUgi);
LOG.info("Application with id " + applicationId.getId() +
" submitted by user " + user);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 762d043..7ebcd65 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -352,16 +352,26 @@ public class RMAppManager implements
EventHandler,
}
}
- @SuppressWarnings("unchecked")
+ @VisibleForTesting
+ @Deprecated
protected void submitApplication(
ApplicationSubmissionContext submissionContext, long submitTime,
String user) throws YarnException {
+submitApplication(submissionContext, submitTime,
+UserGroupInformation.createRemoteUser(user));
+ }
+
+ @VisibleForTesting
+ @SuppressWarnings("unchecked")
+ protected void submitApplication(
+ ApplicationSubmissionContext submissionContext, long submitTime,
+ UserGroupInformation userUgi) throws YarnException {
ApplicationId applicationId = submissionContext.getApplicationId();
// Passing start time as -1. It will be eventually set in RMAppImpl
// constructor.
RMAppImpl application = createAndPopulateNewRMApp(
-
[hadoop] branch trunk updated: YARN-10916. Investigate and simplify GuaranteedOrZeroCapacityOverTimePolicy#computeQueueManagementChanges. Contributed by Szilard Nemeth
This is an automated email from the ASF dual-hosted git repository.
shuzirra pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 20aeb5e YARN-10916. Investigate and simplify
GuaranteedOrZeroCapacityOverTimePolicy#computeQueueManagementChanges.
Contributed by Szilard Nemeth
20aeb5e is described below
commit 20aeb5ecc33f6786f96fb50df22c55aa987fbfc9
Author: Szilard Nemeth <954799+szilard-nem...@users.noreply.github.com>
AuthorDate: Wed Oct 20 15:52:37 2021 +0200
YARN-10916. Investigate and simplify
GuaranteedOrZeroCapacityOverTimePolicy#computeQueueManagementChanges.
Contributed by Szilard Nemeth
---
.../DeactivatedLeafQueuesByLabel.java | 107 +
.../GuaranteedOrZeroCapacityOverTimePolicy.java| 241 -
.../queuemanagement/LeafQueueEntitlements.java | 52 +
.../capacity/queuemanagement/package-info.java}| 29 +--
.../TestDeactivatedLeafQueuesByLabel.java} | 38 ++--
5 files changed, 278 insertions(+), 189 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/DeactivatedLeafQueuesByLabel.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/DeactivatedLeafQueuesByLabel.java
new file mode 100644
index 000..31486a1
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/DeactivatedLeafQueuesByLabel.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.queuemanagement;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
+import org.slf4j.Logger;
+
+import java.util.Map;
+import java.util.Set;
+
+import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON;
+
+public class DeactivatedLeafQueuesByLabel {
+ private String parentQueuePath;
+ private String nodeLabel;
+ private Map deactivatedLeafQueues;
+ private float sumOfChildQueueActivatedCapacity;
+ private float parentAbsoluteCapacity;
+ private float leafQueueTemplateAbsoluteCapacity;
+ private float availableCapacity;
+ private float totalDeactivatedCapacity;
+
+ @VisibleForTesting
+ public DeactivatedLeafQueuesByLabel() {}
+
+ public DeactivatedLeafQueuesByLabel(
+ Map deactivatedLeafQueues,
+ String parentQueuePath,
+ String nodeLabel,
+ float sumOfChildQueueActivatedCapacity,
+ float parentAbsoluteCapacity,
+ float leafQueueTemplateAbsoluteCapacity) {
+this.parentQueuePath = parentQueuePath;
+this.nodeLabel = nodeLabel;
+this.deactivatedLeafQueues = deactivatedLeafQueues;
+this.sumOfChildQueueActivatedCapacity = sumOfChildQueueActivatedCapacity;
+this.parentAbsoluteCapacity = parentAbsoluteCapacity;
+this.leafQueueTemplateAbsoluteCapacity = leafQueueTemplateAbsoluteCapacity;
+
+this.totalDeactivatedCapacity = getTotalDeactivatedCapacity();
+this.availableCapacity = parentAbsoluteCapacity -
sumOfChildQueueActivatedCapacity +
+this.totalDeactivatedCapacity + EPSILON;
+ }
+
+ float getTotalDeactivatedCapacity() {
+float deactivatedCapacity = 0;
+for (Map.Entry deactivatedQueueCapacity :
+deactivatedLeafQueues.entrySet()) {
+ deactivatedCapacity +=
deactivatedQueueCapacity.getValue().getAbsoluteCapacity(nodeLabel);
+}
+return deactivatedCapacity;
+ }
+
+ public Set getQueues() {
+return deactivatedLeafQueues.keySet();
+ }
+
+ public void printToDebug(Logger logger) {
+if (logger.isDebugEnabled()) {
+ logger.debug("Parent queue = {}, nodeLabel = {}, absCapacity = {}, " +
+ "leafQueueAbsoluteCapacity = {}, dea
[hadoop] branch trunk updated: YARN-10949. Simplify AbstractCSQueue#updateMaxAppRelatedField and find a more meaningful name for this method. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
shuzirra pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 35b8441 YARN-10949. Simplify AbstractCSQueue#updateMaxAppRelatedField
and find a more meaningful name for this method. Contributed by Andras Gyori
35b8441 is described below
commit 35b8441fd945766ce7cad87be7209534f9acdad2
Author: Andras Gyori
AuthorDate: Wed Oct 20 12:56:41 2021 +0200
YARN-10949. Simplify AbstractCSQueue#updateMaxAppRelatedField and find a
more meaningful name for this method. Contributed by Andras Gyori
---
.../scheduler/capacity/AbstractCSQueue.java| 47 --
.../capacity/CapacitySchedulerConfiguration.java | 4 ++
.../scheduler/capacity/LeafQueue.java | 58 --
.../scheduler/capacity/TestApplicationLimits.java | 41 ---
.../scheduler/capacity/TestLeafQueue.java | 7 +++
.../scheduler/capacity/TestParentQueue.java| 2 +-
6 files changed, 87 insertions(+), 72 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index e25c694..ca861fb 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -1462,53 +1462,6 @@ public abstract class AbstractCSQueue implements CSQueue
{
configuredMaxResource, parentMaxResource));
}
- void updateMaxAppRelatedField(CapacitySchedulerConfiguration conf,
- LeafQueue leafQueue) {
-int maxApplications = conf.getMaximumApplicationsPerQueue(queuePath);
-int maxGlobalPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue();
-String maxLabel = RMNodeLabelsManager.NO_LABEL;
-
-if (maxApplications < 0) {
- for (String label : configuredNodeLabels) {
-int maxApplicationsByLabel = 0;
-if (maxGlobalPerQueueApps > 0) {
- // In absolute mode, should
- // shrink when change to corresponding label capacity.
- maxApplicationsByLabel = this.capacityConfigType
- != CapacityConfigType.ABSOLUTE_RESOURCE ?
- maxGlobalPerQueueApps :
- (int) (maxGlobalPerQueueApps * queueCapacities
- .getAbsoluteCapacity(label));
-} else {
- maxApplicationsByLabel = (int) (conf.getMaximumSystemApplications()
- * queueCapacities.getAbsoluteCapacity(label));
-}
-if (maxApplicationsByLabel > maxApplications) {
- maxApplications = maxApplicationsByLabel;
- maxLabel = label;
-}
- }
-}
-leafQueue.setMaxApplications(maxApplications);
-
-int maxApplicationsPerUser = Math.min(maxApplications,
-(int) (maxApplications
-* (leafQueue.getUsersManager().getUserLimit() / 100.0f)
-* leafQueue.getUsersManager().getUserLimitFactor()));
-if (leafQueue.getUsersManager().getUserLimitFactor() == -1) {
- maxApplicationsPerUser = maxApplications;
-}
-
-leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser);
-LOG.info("LeafQueue:" + leafQueue.getQueuePath() +
-"update max app related, maxApplications="
-+ maxApplications + ", maxApplicationsPerUser="
-+ maxApplicationsPerUser + ", Abs Cap:" + queueCapacities
-.getAbsoluteCapacity(maxLabel) + ", Cap: " + queueCapacities
-.getCapacity(maxLabel) + ", MaxCap : " + queueCapacities
-.getMaximumCapacity(maxLabel));
- }
-
void deriveCapacityFromAbsoluteConfigurations(String label,
Resource clusterResource) {
// Update capacity with a float calculated from the parent's minResources
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 5e614f1..e7b1cbd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigura
[hadoop] branch trunk updated: YARN-10958. Use correct configuration for Group service init in CSMappingPlacementRule (#3560)
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 414d401 YARN-10958. Use correct configuration for Group service init
in CSMappingPlacementRule (#3560)
414d401 is described below
commit 414d40155cea40ecceab3c927dd9ebfd899ba7a0
Author: Szilard Nemeth <954799+szilard-nem...@users.noreply.github.com>
AuthorDate: Wed Oct 20 10:48:42 2021 +0200
YARN-10958. Use correct configuration for Group service init in
CSMappingPlacementRule (#3560)
* YARN-10958. Initial commit
* Fix javadoc + behaviour
* Fix review comments
* fix checkstyle + blanks
* fix checkstyle + blanks
* Fix checkstyle + blanks
---
.../java/org/apache/hadoop/security/Groups.java| 5 +
.../placement/CSMappingPlacementRule.java | 7 +-
.../csmappingrule/TestCSMappingPlacementRule.java | 137 +
3 files changed, 148 insertions(+), 1 deletion(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index bc2ea45..70c633c 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -493,4 +493,9 @@ public class Groups {
GROUPS = new Groups(conf);
return GROUPS;
}
+
+ @VisibleForTesting
+ public static void reset() {
+GROUPS = null;
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
index ef98c1a..91e0138 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
@@ -133,7 +133,7 @@ public class CSMappingPlacementRule extends PlacementRule {
overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
if (groups == null) {
- groups = Groups.getUserToGroupsMappingService(conf);
+ groups = Groups.getUserToGroupsMappingService(csContext.getConf());
}
MappingRuleValidationContext validationContext = buildValidationContext();
@@ -535,4 +535,9 @@ public class CSMappingPlacementRule extends PlacementRule {
return name;
}
}
+
+ @VisibleForTesting
+ public Groups getGroups() {
+return groups;
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
index 0cf1059..3e614bc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.GroupMappingServiceProvider;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.hadoop.security.Groups;
@@ -51,6 +53,7 @@ import static junit.framework.TestCase.assertNotNull;
import static junit.framework.TestCase.assertNull;
import static junit.framework.TestCase.assertTrue;
import static junit.framework.TestCase.fail;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_GROUP_MAPPING;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -771,4 +774,138 @@ public class TestCSMappingPlacementRule {
"Application should have been placed to
root.groups.sec_dot_test_dot_grp",
