[hadoop] branch trunk updated: YARN-10958. Use correct configuration for Group service init in CSMappingPlacementRule (#3560)

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 414d401  YARN-10958. Use correct configuration for Group service init 
in CSMappingPlacementRule (#3560)
414d401 is described below

commit 414d40155cea40ecceab3c927dd9ebfd899ba7a0
Author: Szilard Nemeth <954799+szilard-nem...@users.noreply.github.com>
AuthorDate: Wed Oct 20 10:48:42 2021 +0200

YARN-10958. Use correct configuration for Group service init in 
CSMappingPlacementRule (#3560)

* YARN-10958. Initial commit

* Fix javadoc + behaviour

* Fix review comments

* fix checkstyle + blanks

* fix checkstyle + blanks

* Fix checkstyle + blanks
---
 .../java/org/apache/hadoop/security/Groups.java|   5 +
 .../placement/CSMappingPlacementRule.java  |   7 +-
 .../csmappingrule/TestCSMappingPlacementRule.java  | 137 +
 3 files changed, 148 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index bc2ea45..70c633c 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -493,4 +493,9 @@ public class Groups {
 GROUPS = new Groups(conf);
 return GROUPS;
   }
+
+  @VisibleForTesting
+  public static void reset() {
+GROUPS = null;
+  }
 }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
index ef98c1a..91e0138 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
@@ -133,7 +133,7 @@ public class CSMappingPlacementRule extends PlacementRule {
 overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
 
 if (groups == null) {
-  groups = Groups.getUserToGroupsMappingService(conf);
+  groups = Groups.getUserToGroupsMappingService(csContext.getConf());
 }
 
 MappingRuleValidationContext validationContext = buildValidationContext();
@@ -535,4 +535,9 @@ public class CSMappingPlacementRule extends PlacementRule {
   return name;
 }
   }
+
+  @VisibleForTesting
+  public Groups getGroups() {
+return groups;
+  }
 }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
index 0cf1059..3e614bc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.GroupMappingServiceProvider;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
 import org.apache.hadoop.security.Groups;
@@ -51,6 +53,7 @@ import static junit.framework.TestCase.assertNotNull;
 import static junit.framework.TestCase.assertNull;
 import static junit.framework.TestCase.assertTrue;
 import static junit.framework.TestCase.fail;
+import static 
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_GROUP_MAPPING;
 import static org.mockito.ArgumentMatchers.isNull;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -771,4 +774,138 @@ public class TestCSMappingPlacementRule {
 "Application should have been placed to 
root.groups.sec_dot_t

[hadoop] branch branch-3.3 updated: YARN-9606. Set sslfactory for AuthenticatedURL() while creating LogsCLI#webServiceClient. Contributed by Bilwa S T.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 99d84b9  YARN-9606. Set sslfactory for AuthenticatedURL() while 
creating LogsCLI#webServiceClient. Contributed by Bilwa S T.
99d84b9 is described below

commit 99d84b941bdb1483303d9c80cf63b0288b2b7f08
Author: Peter Bacsko 
AuthorDate: Thu Sep 23 13:58:56 2021 +0200

YARN-9606. Set sslfactory for AuthenticatedURL() while creating 
LogsCLI#webServiceClient. Contributed by Bilwa S T.
---
 .../org/apache/hadoop/yarn/client/cli/LogsCLI.java |  25 +---
 .../apache/hadoop/yarn/client/cli/TestLogsCLI.java | 132 ++---
 .../hadoop/yarn/webapp/util}/WebServiceClient.java |  12 +-
 .../yarn/webapp/util}/TestWebServiceClient.java|   4 +-
 .../apache/hadoop/yarn/server/router/Router.java   |   2 +-
 .../server/router/webapp/RouterWebServiceUtil.java |   2 +-
 6 files changed, 77 insertions(+), 100 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 6d2f0bb..062d279 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -27,17 +27,13 @@ import com.sun.jersey.api.client.UniformInterfaceException;
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.api.client.WebResource.Builder;
 import com.sun.jersey.api.client.filter.ClientFilter;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintStream;
 import java.net.ConnectException;
-import java.net.HttpURLConnection;
 import java.net.SocketException;
 import java.net.SocketTimeoutException;
-import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -68,8 +64,6 @@ import 
org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
-import 
org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -89,6 +83,7 @@ import org.apache.hadoop.yarn.logaggregation.LogToolUtils;
 import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
 import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.util.WebServiceClient;
 import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
 import org.codehaus.jettison.json.JSONArray;
 import org.codehaus.jettison.json.JSONException;
@@ -140,21 +135,7 @@ public class LogsCLI extends Configured implements Tool {
   @Override
   public int run(String[] args) throws Exception {
 try {
-  webServiceClient = new Client(new URLConnectionClientHandler(
-  new HttpURLConnectionFactory() {
-  @Override
-  public HttpURLConnection getHttpURLConnection(URL url)
-  throws IOException {
-AuthenticatedURL.Token token = new AuthenticatedURL.Token();
-HttpURLConnection conn = null;
-try {
-  conn = new AuthenticatedURL().openConnection(url, token);
-} catch (AuthenticationException e) {
-  throw new IOException(e);
-}
-return conn;
-  }
-}));
+  webServiceClient = WebServiceClient.getWebServiceClient().createClient();
   return runCommand(args);
 } finally {
   if (yarnClient != null) {
@@ -418,7 +399,9 @@ public class LogsCLI extends Configured implements Tool {
 Configuration conf = new YarnConfiguration();
 LogsCLI logDumper = new LogsCLI();
 logDumper.setConf(conf);
+WebServiceClient.initialize(conf);
 int exitCode = logDumper.run(args);
+WebServiceClient.destroy();
 System.exit(exitCode);
   }
 
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index a5549c5..3844c3f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org

[hadoop] branch trunk updated: YARN-10780. Optimise retrieval of configured node labels in CS queues. Contributed by Andras Gyori.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 0934e78  YARN-10780. Optimise retrieval of configured node labels in 
CS queues. Contributed by Andras Gyori.
0934e78 is described below

commit 0934e783cf687eee4f5f23e65500b91d18b96f65
Author: Peter Bacsko 
AuthorDate: Thu Jun 24 20:15:10 2021 +0200

YARN-10780. Optimise retrieval of configured node labels in CS queues. 
Contributed by Andras Gyori.
---
 .../scheduler/capacity/AbstractCSQueue.java| 40 ---
 .../scheduler/capacity/AutoCreatedLeafQueue.java   | 20 +-
 .../scheduler/capacity/CSQueueUtils.java   |  9 ++-
 .../capacity/CapacitySchedulerConfiguration.java   | 36 ++
 .../capacity/CapacitySchedulerQueueManager.java| 13 
 .../scheduler/capacity/ConfiguredNodeLabels.java   | 77 ++
 .../scheduler/capacity/ManagedParentQueue.java | 16 +++--
 .../scheduler/capacity/ParentQueue.java|  5 +-
 .../TestCapacitySchedulerNewQueueAutoCreation.java | 15 +
 .../scheduler/capacity/TestReservationQueue.java   |  5 ++
 10 files changed, 210 insertions(+), 26 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 94ade5e..2f6ca5a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -76,6 +76,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import static 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
 import static 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.UNDEFINED;
 
 public abstract class AbstractCSQueue implements CSQueue {
@@ -95,6 +96,7 @@ public abstract class AbstractCSQueue implements CSQueue {
 
   final ResourceCalculator resourceCalculator;
   Set accessibleLabels;
+  protected Set configuredNodeLabels;
   Set resourceTypes;
   final RMNodeLabelsManager labelManager;
   String defaultLabelExpression;
@@ -208,7 +210,7 @@ public abstract class AbstractCSQueue implements CSQueue {
   protected void setupConfigurableCapacities(
   CapacitySchedulerConfiguration configuration) {
 CSQueueUtils.loadCapacitiesByLabelsFromConf(getQueuePath(), 
queueCapacities,
-configuration);
+configuration, configuredNodeLabels);
   }
 
   @Override
@@ -360,7 +362,7 @@ public abstract class AbstractCSQueue implements CSQueue {
 
 writeLock.lock();
 try {
-  if (isDynamicQueue()) {
+  if (isDynamicQueue() || this instanceof AbstractAutoCreatedLeafQueue) {
 setDynamicQueueProperties(configuration);
   }
   // get labels
@@ -386,6 +388,17 @@ public abstract class AbstractCSQueue implements CSQueue {
 this.defaultLabelExpression = parent.getDefaultNodeLabelExpression();
   }
 
+  if (csContext.getCapacitySchedulerQueueManager() != null
+  && csContext.getCapacitySchedulerQueueManager()
+  .getConfiguredNodeLabels() != null) {
+this.configuredNodeLabels = 
csContext.getCapacitySchedulerQueueManager()
+.getConfiguredNodeLabels().getLabelsByQueue(getQueuePath());
+  } else {
+// Fallback to suboptimal but correct logic
+this.configuredNodeLabels = csContext.getConfiguration()
+.getConfiguredNodeLabels(queuePath);
+  }
+
   // After we setup labels, we can setup capacities
   setupConfigurableCapacities(configuration);
   updateAbsoluteCapacities();
@@ -487,6 +500,19 @@ public abstract class AbstractCSQueue implements CSQueue {
 if (getParent() instanceof ParentQueue) {
   ((ParentQueue) getParent()).getAutoCreatedQueueTemplate()
   .setTemplateEntriesForChild(configuration, getQueuePath());
+
+  String parentTemplate = String.format("%s.%s", 
getParent().getQueuePath(),
+  AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX);
+  parentTemplate = parentTemplate.substring(0, parentTemplate.lastIndexOf(
+  DOT));
+  Set parentNodeLabels = csContext
+  .getCapacitySchedulerQueueManager().getConfiguredNodeLabels()
+ 

[hadoop] branch branch-3.3 updated: YARN-10120. In Federation Router Nodes/Applications/About pages throws 500 exception when https is enabled. Contributed by Bilwa S T.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new a45d643  YARN-10120. In Federation Router Nodes/Applications/About 
pages throws 500 exception when https is enabled. Contributed by Bilwa S T.
a45d643 is described below

commit a45d6437f5f3fc044418a969dea286de0e44d72c
Author: Peter Bacsko 
AuthorDate: Wed May 19 15:07:47 2021 +0200

YARN-10120. In Federation Router Nodes/Applications/About pages throws 500 
exception when https is enabled. Contributed by Bilwa S T.
---
 .../hadoop-yarn-server-common/pom.xml  |   5 +
 .../yarn/server/webapp/WebServiceClient.java   | 131 +
 .../yarn/server/webapp/TestWebServiceClient.java   | 101 
 .../apache/hadoop/yarn/server/router/Router.java   |   3 +
 .../yarn/server/router/webapp/AboutBlock.java  |   2 +-
 .../yarn/server/router/webapp/AppsBlock.java   |   2 +-
 .../webapp/DefaultRequestInterceptorREST.java  | 102 
 .../yarn/server/router/webapp/NodesBlock.java  |   2 +-
 .../server/router/webapp/RouterWebServiceUtil.java |  23 +++-
 9 files changed, 315 insertions(+), 56 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 381e53d..151a9c9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -75,6 +75,11 @@
   org.apache.hadoop.thirdparty
   hadoop-shaded-guava
 
+
+  org.bouncycastle
+  bcprov-jdk15on
+  test
+
 
 
 
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServiceClient.java
new file mode 100644
index 000..bcd58e5
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServiceClient.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.webapp;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import 
org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+import 
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+
+/**
+ * Utility for handling Web client.
+ *
+ */
+public class WebServiceClient {
+  private static SSLFactory sslFactory = null;
+  private static volatile WebServiceClient instance = null;
+  private static boolean isHttps = false;
+
+  /**
+   * Construct a new WebServiceClient based on the configuration. It will try 
to
+   * load SSL certificates when it is specified.
+   *
+   * @param conf Configuration object
+   * @throws Exception if creation of SSLFactory fails
+   */
+  public static void initialize(Configuration conf) throws Exception {
+if (instance == null) {
+  synchronized (WebServiceClient.class) {
+if (instance == null) {
+  isHttps = YarnConfiguration.useHttps(conf);
+  if (isHttps) {
+createSSLFactory(conf);
+  }
+  instance = new WebServiceClient();
+}
+  }
+}
+  

[hadoop] branch branch-3.3 updated: YARN-10258. Add metrics for 'ApplicationsRunning' in NodeManager. Contributed by ANANDA G B.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 864b710  YARN-10258. Add metrics for 'ApplicationsRunning' in 
NodeManager. Contributed by ANANDA G B.
864b710 is described below

commit 864b710787b6e418abcf2fb535b507bf5336f36f
Author: Peter Bacsko 
AuthorDate: Wed May 19 10:31:57 2021 +0200

YARN-10258. Add metrics for 'ApplicationsRunning' in NodeManager. 
Contributed by ANANDA G B.
---
 .../nodemanager/containermanager/ContainerManagerImpl.java |  2 ++
 .../containermanager/application/ApplicationImpl.java  |  3 +++
 .../server/nodemanager/metrics/NodeManagerMetrics.java | 10 ++
 .../containermanager/TestContainerManagerRecovery.java |  6 +++---
 .../server/nodemanager/metrics/TestNodeManagerMetrics.java | 14 +-
 5 files changed, 27 insertions(+), 8 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 5eb36ba..ee1a5bf 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -442,6 +442,7 @@ public class ContainerManagerImpl extends CompositeService 
implements
 ApplicationImpl app = new ApplicationImpl(dispatcher, p.getUser(), fc,
 appId, creds, context, p.getAppLogAggregationInitedTime());
 context.getApplications().put(appId, app);
+metrics.runningApplication();
 app.handle(new ApplicationInitEvent(appId, acls, logAggregationContext));
   }
 
@@ -1137,6 +1138,7 @@ public class ContainerManagerImpl extends 
CompositeService implements
   applicationID, credentials, context);
   if (context.getApplications().putIfAbsent(applicationID,
   application) == null) {
+metrics.runningApplication();
 LOG.info("Creating a new application reference for app "
 + applicationID);
 LogAggregationContext logAggregationContext =
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index 8fe9651..d420974 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -623,6 +623,9 @@ public class ApplicationImpl implements Application {
 public void transition(ApplicationImpl app, ApplicationEvent event) {
   ApplicationId appId = event.getApplicationID();
   app.context.getApplications().remove(appId);
+  if (null != app.context.getNodeManagerMetrics()) {
+app.context.getNodeManagerMetrics().endRunningApplication();
+  }
   app.aclsManager.removeApplication(appId);
   try {
 app.context.getNMStateStore().removeApplication(appId);
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
index 848b944..a469f65 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
@@ -100,6 +100,8 @@ public class NodeManagerMetrics {
   MutableGaugeFloat nodeCpuUtilization;
   @Metric("Current GPU utilization")
   MutableGaugeFloat nodeGpuUtilization;
+  @Metric("Current running 

[hadoop] branch branch-3.3 updated: YARN-9615. addendum commit, removing accedentially committed patch file

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 27749ce  YARN-9615. addendum commit, removing accedentially committed 
patch file
27749ce is described below

commit 27749ce85d306ce765efedcf0154c179e70e91e9
Author: Peter Bacsko 
AuthorDate: Tue May 18 22:17:20 2021 +0200

YARN-9615. addendum commit, removing accedentially committed patch file
---
 YARN-9615-branch-3.3-001.patch.1 | 732 ---
 1 file changed, 732 deletions(-)

diff --git a/YARN-9615-branch-3.3-001.patch.1 b/YARN-9615-branch-3.3-001.patch.1
deleted file mode 100644
index 80e9ca9..000
--- a/YARN-9615-branch-3.3-001.patch.1
+++ /dev/null
@@ -1,732 +0,0 @@
-diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
-index f9deab06ff2..667515d00c1 100644
 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
-+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
-@@ -26,6 +26,9 @@
- import java.util.concurrent.BlockingQueue;
- import java.util.concurrent.LinkedBlockingQueue;
- 
-+import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
-+import org.apache.hadoop.yarn.util.Clock;
-+import org.apache.hadoop.yarn.util.MonotonicClock;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
- import org.slf4j.Marker;
-@@ -85,6 +88,11 @@
-   protected final Map, EventHandler> eventDispatchers;
-   private boolean exitOnDispatchException = true;
- 
-+  private Map,
-+  EventTypeMetrics> eventTypeMetricsMap;
-+
-+  private Clock clock = new MonotonicClock();
-+
-   /**
-* The thread name for dispatcher.
-*/
-@@ -98,6 +106,8 @@ public AsyncDispatcher(BlockingQueue eventQueue) {
- super("Dispatcher");
- this.eventQueue = eventQueue;
- this.eventDispatchers = new HashMap, 
EventHandler>();
-+this.eventTypeMetricsMap = new HashMap,
-+EventTypeMetrics>();
-   }
- 
-   /**
-@@ -135,7 +145,16 @@ public void run() {
- return;
-   }
-   if (event != null) {
--dispatch(event);
-+if (eventTypeMetricsMap.
-+get(event.getType().getDeclaringClass()) != null) {
-+  long startTime = clock.getTime();
-+  dispatch(event);
-+  eventTypeMetricsMap.get(event.getType().getDeclaringClass())
-+  .increment(event.getType(),
-+  clock.getTime() - startTime);
-+} else {
-+  dispatch(event);
-+}
- if (printTrigger) {
-   //Log the latest dispatch event type
-   // may cause the too many events queued
-@@ -369,4 +388,9 @@ protected boolean isDrained() {
-   protected boolean isStopped() {
- return stopped;
-   }
-+
-+  public void addMetrics(EventTypeMetrics metrics,
-+  Class eventClass) {
-+eventTypeMetricsMap.put(eventClass, metrics);
-+  }
- }
-diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
-index cadb73663a0..849bb402d87 100644
 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
-+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
-@@ -19,6 +19,9 @@
- package org.apache.hadoop.yarn.event;
- 
- import 
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-+import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
-+import org.apache.hadoop.yarn.util.Clock;
-+import org.apache.hadoop.yarn.util.MonotonicClock;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
- import org.slf4j.Marker;
-@@ -47,12 +50,15 @@
-   private final Thread eventProcessor;
-   private volatile boolean stopped = false;
-   private boolean shouldExitOnError = true;
-+  private EventTypeMetrics metrics;
- 
-   private static final Logger LOG =
-   LoggerFactory.getLogger(EventDispatcher.class);
-   private static final Marker FATAL =
-   MarkerFactory.getMarker("FATAL");
- 
-+  private Clock clock = new MonotonicClock();
-+
-   private final class EventProcessor implements Runnable {
- @Override
- public void run() {
-@@ -68,7 +74,14 @@ public void run() {
- }
- 
- try {
--  handler.handle(event);
-+  if (metrics != null

[hadoop] branch trunk updated: YARN-10258. Add metrics for 'ApplicationsRunning' in NodeManager. Contributed by ANANDA G B.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new eb72628  YARN-10258. Add metrics for 'ApplicationsRunning' in 
NodeManager. Contributed by ANANDA G B.
eb72628 is described below

commit eb72628e151b70c6513723f37f87c7855a624876
Author: Peter Bacsko 
AuthorDate: Mon May 17 21:07:29 2021 +0200

YARN-10258. Add metrics for 'ApplicationsRunning' in NodeManager. 
Contributed by ANANDA G B.
---
 .../nodemanager/containermanager/ContainerManagerImpl.java |  2 ++
 .../containermanager/application/ApplicationImpl.java  |  3 +++
 .../server/nodemanager/metrics/NodeManagerMetrics.java | 10 ++
 .../containermanager/TestContainerManagerRecovery.java |  6 +++---
 .../server/nodemanager/metrics/TestNodeManagerMetrics.java | 14 +-
 5 files changed, 27 insertions(+), 8 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 5eb36ba..ee1a5bf 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -442,6 +442,7 @@ public class ContainerManagerImpl extends CompositeService 
implements
 ApplicationImpl app = new ApplicationImpl(dispatcher, p.getUser(), fc,
 appId, creds, context, p.getAppLogAggregationInitedTime());
 context.getApplications().put(appId, app);
+metrics.runningApplication();
 app.handle(new ApplicationInitEvent(appId, acls, logAggregationContext));
   }
 
@@ -1137,6 +1138,7 @@ public class ContainerManagerImpl extends 
CompositeService implements
   applicationID, credentials, context);
   if (context.getApplications().putIfAbsent(applicationID,
   application) == null) {
+metrics.runningApplication();
 LOG.info("Creating a new application reference for app "
 + applicationID);
 LogAggregationContext logAggregationContext =
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index 8fe9651..d420974 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -623,6 +623,9 @@ public class ApplicationImpl implements Application {
 public void transition(ApplicationImpl app, ApplicationEvent event) {
   ApplicationId appId = event.getApplicationID();
   app.context.getApplications().remove(appId);
+  if (null != app.context.getNodeManagerMetrics()) {
+app.context.getNodeManagerMetrics().endRunningApplication();
+  }
   app.aclsManager.removeApplication(appId);
   try {
 app.context.getNMStateStore().removeApplication(appId);
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
index 848b944..a469f65 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
@@ -100,6 +100,8 @@ public class NodeManagerMetrics {
   MutableGaugeFloat nodeCpuUtilization;
   @Metric("Current GPU utilization")
   MutableGaugeFloat nodeGpuUtilization;
+  @Metric("Current running apps")
+  Mutab

[hadoop] branch trunk updated: YARN-10763. Add the number of containers assigned per second metrics to ClusterMetrics. Contributed by chaosju.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 8891e5c  YARN-10763. Add the number of containers assigned per second 
metrics to ClusterMetrics. Contributed by chaosju.
8891e5c is described below

commit 8891e5c028299803410b93f52e0a61df16f84d62
Author: Peter Bacsko 
AuthorDate: Mon May 17 13:30:12 2021 +0200

YARN-10763. Add the number of containers assigned per second metrics to 
ClusterMetrics. Contributed by chaosju.
---
 .../server/resourcemanager/ClusterMetrics.java | 37 ++
 .../scheduler/AppSchedulingInfo.java   |  2 ++
 .../webapp/dao/ClusterMetricsInfo.java |  8 +
 .../server/resourcemanager/TestClusterMetrics.java | 17 ++
 .../resourcemanager/webapp/TestRMWebServices.java  |  2 +-
 5 files changed, 65 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
index a02eeef..67a3a62 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java
@@ -21,8 +21,12 @@ package org.apache.hadoop.yarn.server.resourcemanager;
 import static org.apache.hadoop.metrics2.lib.Interns.info;
 
 import java.util.Map;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
+import 
org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.metrics2.MetricsInfo;
 import org.apache.hadoop.metrics2.MetricsSystem;
@@ -66,6 +70,8 @@ public class ClusterMetrics {
 rmEventProcCPUAvg;
   @Metric("RM Event Processor CPU Usage 60 second Max") MutableGaugeLong
 rmEventProcCPUMax;
+  @Metric("# of Containers assigned in the last second") MutableGaugeInt
+containerAssignedPerSecond;
 
   private boolean rmEventProcMonitorEnable = false;
 
@@ -85,6 +91,22 @@ public class ClusterMetrics {
   private static volatile ClusterMetrics INSTANCE = null;
   private static MetricsRegistry registry;
 
+  private AtomicInteger numContainersAssigned =  new AtomicInteger(0);
+  private ScheduledThreadPoolExecutor assignCounterExecutor;
+
+  ClusterMetrics() {
+assignCounterExecutor  = new ScheduledThreadPoolExecutor(1,
+new ThreadFactoryBuilder().
+setDaemon(true).setNameFormat("ContainerAssignmentCounterThread").
+build());
+assignCounterExecutor.scheduleAtFixedRate(new Runnable() {
+  @Override
+  public void run() {
+containerAssignedPerSecond.set(numContainersAssigned.getAndSet(0));
+  }
+}, 1, 1, TimeUnit.SECONDS);
+  }
+
   public static ClusterMetrics getMetrics() {
 if(!isInitialized.get()){
   synchronized (ClusterMetrics.class) {
@@ -120,6 +142,9 @@ public class ClusterMetrics {
 
   @VisibleForTesting
   public synchronized static void destroy() {
+if (INSTANCE != null && INSTANCE.getAssignCounterExecutor() != null) {
+  INSTANCE.getAssignCounterExecutor().shutdownNow();
+}
 isInitialized.set(false);
 INSTANCE = null;
   }
@@ -319,4 +344,16 @@ public class ClusterMetrics {
   public void incrUtilizedVirtualCores(long delta) {
 utilizedVirtualCores.incr(delta);
   }
+
+  public int getContainerAssignedPerSecond() {
+return containerAssignedPerSecond.value();
+  }
+
+  public void incrNumContainerAssigned() {
+numContainersAssigned.incrementAndGet();
+  }
+
+  private ScheduledThreadPoolExecutor getAssignCounterExecutor(){
+return assignCounterExecutor;
+  }
 }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index a3b235b..2b2f832 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ 
b/hadoop-yarn-

[hadoop] branch trunk updated: YARN-10571. Refactor dynamic queue handling logic. Contributed by Andras Gyori.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 626be24  YARN-10571. Refactor dynamic queue handling logic. 
Contributed by Andras Gyori.
626be24 is described below

commit 626be24c3ee1180fdfbd5dc968c70271770f21cc
Author: Peter Bacsko 
AuthorDate: Wed May 12 14:54:47 2021 +0200

YARN-10571. Refactor dynamic queue handling logic. Contributed by Andras 
Gyori.
---
 .../scheduler/capacity/CapacityScheduler.java  | 101 +
 .../CapacitySchedulerAutoQueueHandler.java | 140 -
 .../capacity/CapacitySchedulerQueueManager.java| 227 +
 .../TestCapacitySchedulerNewQueueAutoCreation.java |  41 +++-
 ...estRMWebServicesCapacitySchedDynamicConfig.java |  14 +-
 5 files changed, 274 insertions(+), 249 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 37acb60..1e81f67 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -233,8 +233,6 @@ public class CapacityScheduler extends
   private AppPriorityACLsManager appPriorityACLManager;
   private boolean multiNodePlacementEnabled;
 
-  private CapacitySchedulerAutoQueueHandler autoQueueHandler;
-
   private boolean printedVerboseLoggingForAsyncScheduling;
 
   /**
@@ -343,9 +341,6 @@ public class CapacityScheduler extends
   this.labelManager, this.appPriorityACLManager);
   this.queueManager.setCapacitySchedulerContext(this);
 
-  this.autoQueueHandler = new CapacitySchedulerAutoQueueHandler(
-  this.queueManager);
-
   this.workflowPriorityMappingsMgr = new WorkflowPriorityMappingsManager();
 
   this.activitiesManager = new ActivitiesManager(rmContext);
@@ -970,7 +965,8 @@ public class CapacityScheduler extends
 
   if (fallbackContext.hasParentQueue()) {
 try {
-  return autoCreateLeafQueue(fallbackContext);
+  writeLock.lock();
+  return queueManager.createQueue(fallbackContext);
 } catch (YarnException | IOException e) {
   // A null queue is expected if the placementContext is null. In order
   // not to disrupt the control flow, if we fail to auto create a 
queue,
@@ -1007,6 +1003,8 @@ public class CapacityScheduler extends
 new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED,
 message));
   }
+} finally {
+  writeLock.unlock();
 }
   }
 }
@@ -2557,30 +2555,7 @@ public class CapacityScheduler extends
   throws SchedulerDynamicEditException {
 writeLock.lock();
 try {
-  LOG.info("Removing queue: " + queueName);
-  CSQueue q = this.getQueue(queueName);
-  if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(
-  q.getClass( {
-throw new SchedulerDynamicEditException(
-"The queue that we are asked " + "to remove (" + queueName
-+ ") is not a AutoCreatedLeafQueue or ReservationQueue");
-  }
-  AbstractAutoCreatedLeafQueue disposableLeafQueue =
-  (AbstractAutoCreatedLeafQueue) q;
-  // at this point we should have no more apps
-  if (disposableLeafQueue.getNumApplications() > 0) {
-throw new SchedulerDynamicEditException(
-"The queue " + queueName + " is not empty " + disposableLeafQueue
-.getApplications().size() + " active apps "
-+ disposableLeafQueue.getPendingApplications().size()
-+ " pending apps");
-  }
-
-  ((AbstractManagedParentQueue) disposableLeafQueue.getParent())
-  .removeChildQueue(q);
-  this.queueManager.removeQueue(queueName);
-  LOG.info(
-  "Removal of AutoCreatedLeafQueue " + queueName + " has succeeded");
+  queueManager.removeLegacyDynamicQueue(queueName);
 } finally {
   writeLock.unlock();
 }
@@ -2629,34 +2604,7 @@ public class CapacityScheduler extends
   throws SchedulerDynamicEditException, IOException {
 writeLock.lock();
 try {
-  if (queue == null) {
-thro

[hadoop] branch branch-3.3 updated: YARN-9615. Add dispatcher metrics to RM. Contributed by Jonathan Hung and Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 051a506  YARN-9615. Add dispatcher metrics to RM. Contributed by 
Jonathan Hung and Qi Zhu.
051a506 is described below

commit 051a5068dd6d505f0b4748f103a13c4938d61cc0
Author: Peter Bacsko 
AuthorDate: Tue May 11 19:23:45 2021 +0200

YARN-9615. Add dispatcher metrics to RM. Contributed by Jonathan Hung and 
Qi Zhu.
---
 YARN-9615-branch-3.3-001.patch.1   | 732 +
 .../apache/hadoop/yarn/event/AsyncDispatcher.java  |  26 +-
 .../apache/hadoop/yarn/event/EventDispatcher.java  |  21 +-
 .../yarn/metrics/DisableEventTypeMetrics.java  |  42 ++
 .../hadoop/yarn/metrics/EventTypeMetrics.java  |  32 +
 .../yarn/metrics/GenericEventTypeMetrics.java  | 160 +
 .../hadoop/yarn/event/TestAsyncDispatcher.java | 182 -
 .../GenericEventTypeMetricsManager.java|  42 ++
 .../server/resourcemanager/ResourceManager.java|  11 +-
 .../resourcemanager/TestResourceManager.java   |   2 +
 .../scheduler/TestSchedulerHealth.java |   2 +
 .../scheduler/capacity/TestCapacityScheduler.java  |   1 +
 12 files changed, 1248 insertions(+), 5 deletions(-)

diff --git a/YARN-9615-branch-3.3-001.patch.1 b/YARN-9615-branch-3.3-001.patch.1
new file mode 100644
index 000..80e9ca9
--- /dev/null
+++ b/YARN-9615-branch-3.3-001.patch.1
@@ -0,0 +1,732 @@
+diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+index f9deab06ff2..667515d00c1 100644
+--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+@@ -26,6 +26,9 @@
+ import java.util.concurrent.BlockingQueue;
+ import java.util.concurrent.LinkedBlockingQueue;
+ 
++import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
++import org.apache.hadoop.yarn.util.Clock;
++import org.apache.hadoop.yarn.util.MonotonicClock;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.slf4j.Marker;
+@@ -85,6 +88,11 @@
+   protected final Map, EventHandler> eventDispatchers;
+   private boolean exitOnDispatchException = true;
+ 
++  private Map,
++  EventTypeMetrics> eventTypeMetricsMap;
++
++  private Clock clock = new MonotonicClock();
++
+   /**
+* The thread name for dispatcher.
+*/
+@@ -98,6 +106,8 @@ public AsyncDispatcher(BlockingQueue eventQueue) {
+ super("Dispatcher");
+ this.eventQueue = eventQueue;
+ this.eventDispatchers = new HashMap, 
EventHandler>();
++this.eventTypeMetricsMap = new HashMap,
++EventTypeMetrics>();
+   }
+ 
+   /**
+@@ -135,7 +145,16 @@ public void run() {
+ return;
+   }
+   if (event != null) {
+-dispatch(event);
++if (eventTypeMetricsMap.
++get(event.getType().getDeclaringClass()) != null) {
++  long startTime = clock.getTime();
++  dispatch(event);
++  eventTypeMetricsMap.get(event.getType().getDeclaringClass())
++  .increment(event.getType(),
++  clock.getTime() - startTime);
++} else {
++  dispatch(event);
++}
+ if (printTrigger) {
+   //Log the latest dispatch event type
+   // may cause the too many events queued
+@@ -369,4 +388,9 @@ protected boolean isDrained() {
+   protected boolean isStopped() {
+ return stopped;
+   }
++
++  public void addMetrics(EventTypeMetrics metrics,
++  Class eventClass) {
++eventTypeMetricsMap.put(eventClass, metrics);
++  }
+ }
+diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
+index cadb73663a0..849bb402d87 100644
+--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
+@@ -19,6 +19,9 @@
+ package org.apache.hadoop.yarn.event;
+ 
+ import 
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
++import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
++import org.apache.hadoop.yarn.util.Clock;
++import org.apache.hadoop.yarn.util.MonotonicClock;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ i

[hadoop] branch branch-3.1 updated: YARN-10642. Race condition: AsyncDispatcher can get stuck by the changes introduced in YARN-8995. Contributed by zhengchenyu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.1 by this push:
 new 4cd1596  YARN-10642. Race condition: AsyncDispatcher can get stuck by 
the changes introduced in YARN-8995. Contributed by zhengchenyu.
4cd1596 is described below

commit 4cd1596fe7c8d80d7124a517ea6cdc3b3df90292
Author: Peter Bacsko 
AuthorDate: Tue May 11 15:18:16 2021 +0200

YARN-10642. Race condition: AsyncDispatcher can get stuck by the changes 
introduced in YARN-8995. Contributed by zhengchenyu.
---
 .../apache/hadoop/yarn/event/AsyncDispatcher.java  | 19 ---
 .../hadoop/yarn/event/TestAsyncDispatcher.java | 58 +-
 2 files changed, 68 insertions(+), 9 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 81448c5..acfba27 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -20,11 +20,11 @@ package org.apache.hadoop.yarn.event;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.stream.Collectors;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -265,11 +265,16 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   }
 
   class GenericEventHandler implements EventHandler {
-private void printEventQueueDetails(BlockingQueue queue) {
-  Map counterMap = eventQueue.stream().
-  collect(Collectors.
-  groupingBy(e -> e.getType(), Collectors.counting())
-  );
+private void printEventQueueDetails() {
+  Iterator iterator = eventQueue.iterator();
+  Map counterMap = new HashMap<>();
+  while (iterator.hasNext()) {
+Enum eventType = iterator.next().getType();
+if (!counterMap.containsKey(eventType)) {
+  counterMap.put(eventType, 0L);
+}
+counterMap.put(eventType, counterMap.get(eventType) + 1);
+  }
   for (Map.Entry entry : counterMap.entrySet()) {
 long num = entry.getValue();
 LOG.info("Event type: " + entry.getKey()
@@ -292,7 +297,7 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   if (qSize != 0 && qSize % detailsInterval == 0
   && lastEventDetailsQueueSizeLogged != qSize) {
 lastEventDetailsQueueSizeLogged = qSize;
-printEventQueueDetails(eventQueue);
+printEventQueueDetails();
 printTrigger = true;
   }
   int remCapacity = eventQueue.remainingCapacity();
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
index 53e2d05..88855fc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
@@ -97,12 +97,23 @@ public class TestAsyncDispatcher {
   }
 
   private static class TestHandler implements EventHandler {
+
+private long sleepTime = 1500;
+
+TestHandler() {
+}
+
+TestHandler(long sleepTime) {
+  this.sleepTime = sleepTime;
+}
+
 @Override
 public void handle(Event event) {
   try {
 // As long as 1 events queued
-Thread.sleep(1500);
-  } catch (InterruptedException e) {}
+Thread.sleep(this.sleepTime);
+  } catch (InterruptedException e) {
+  }
 }
   }
 
@@ -170,11 +181,54 @@ public class TestAsyncDispatcher {
   //Make sure more than one event to take
   verify(log, atLeastOnce()).
   info("Latest dispatch event type: TestEventType");
+} finally {
+  //... restore logger object
+  logger.set(null, oldLog);
   dispatcher.stop();
+}
+  }
+
+  //Test print dispatcher details when the blocking queue is heavy
+  @Test(timeout = 6)
+  public void testPrintDispatcherEventDetailsAvoidDeadLoop() throws Exception {
+for (int i = 0; i < 5; i++) {
+  testPrintDispatcherEventDetailsAvoidDeadLoopInternal();
+}
+  }
+
+  public void testPrintDispatcherEventDetail

[hadoop] branch trunk updated: YARN-10739. GenericEventHandler.printEventQueueDetails causes RM recovery to take too much time. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new a967ab0  YARN-10739. GenericEventHandler.printEventQueueDetails causes 
RM recovery to take too much time. Contributed by Qi Zhu.
a967ab0 is described below

commit a967ab06f2e802f30ed34eb4ceb2742e4df8be52
Author: Peter Bacsko 
AuthorDate: Tue Apr 27 14:01:53 2021 +0200

YARN-10739. GenericEventHandler.printEventQueueDetails causes RM recovery 
to take too much time. Contributed by Qi Zhu.
---
 .../org/apache/hadoop/yarn/event/AsyncDispatcher.java  | 18 +-
 .../apache/hadoop/yarn/event/TestAsyncDispatcher.java  |  2 --
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 667515d..ba6bb43 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -25,7 +25,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
+import 
org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.MonotonicClock;
@@ -93,6 +97,8 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
 
   private Clock clock = new MonotonicClock();
 
+  private ThreadPoolExecutor printEventDetailsExecutor;
+
   /**
* The thread name for dispatcher.
*/
@@ -179,6 +185,15 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
 YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD,
 YarnConfiguration.
 DEFAULT_YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD);
+
+ThreadFactory threadFactory = new ThreadFactoryBuilder()
+.setNameFormat("PrintEventDetailsThread #%d")
+.build();
+// Thread pool for async print event details,
+// to prevent wasting too much time for RM.
+printEventDetailsExecutor = new ThreadPoolExecutor(
+1, 5, 10, TimeUnit.SECONDS,
+new LinkedBlockingQueue<>(), threadFactory);
   }
 
   @Override
@@ -222,6 +237,7 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
 LOG.warn("Interrupted Exception while stopping", ie);
   }
 }
+printEventDetailsExecutor.shutdownNow();
 
 // stop all the components
 super.serviceStop();
@@ -319,7 +335,7 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   if (qSize != 0 && qSize % detailsInterval == 0
   && lastEventDetailsQueueSizeLogged != qSize) {
 lastEventDetailsQueueSizeLogged = qSize;
-printEventQueueDetails();
+printEventDetailsExecutor.submit(this::printEventQueueDetails);
 printTrigger = true;
   }
   int remCapacity = eventQueue.remainingCapacity();
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
index 7d2572a..8b2dfa0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
@@ -187,8 +187,6 @@ public class TestAsyncDispatcher {
 when(event.getType()).thenReturn(TestEnum.TestEventType);
 dispatcher.getEventHandler().handle(event);
   }
-  verify(log, atLeastOnce()).info("Event type: TestEventType, " +
-  "Event record counter: 5000");
   Thread.sleep(2000);
   //Make sure more than one event to take
   verify(log, atLeastOnce()).

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10637. fs2cs: add queue autorefresh policy during conversion. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 9166bfe  YARN-10637. fs2cs: add queue autorefresh policy during 
conversion. Contributed by Qi Zhu.
9166bfe is described below

commit 9166bfeb74df249c6e5e5a4aca50d3ded81303b4
Author: Peter Bacsko 
AuthorDate: Sat Apr 24 10:10:10 2021 +0200

YARN-10637. fs2cs: add queue autorefresh policy during conversion. 
Contributed by Qi Zhu.
---
 .../scheduler/fair/converter/FSYarnSiteConverter.java  |  6 ++
 .../scheduler/fair/converter/TestFSYarnSiteConverter.java  | 14 ++
 2 files changed, 20 insertions(+)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java
index 401c056c..4f9029a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java
@@ -24,6 +24,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.Proportion
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueDeletionPolicy;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueConfigurationAutoRefreshPolicy;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
 import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
 
@@ -111,6 +112,11 @@ public class FSYarnSiteConverter {
   CapacitySchedulerConfiguration.ASSIGN_MULTIPLE_ENABLED, false);
 }
 
+// Make auto cs conf refresh enabled.
+yarnSiteConfig.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
+addMonitorPolicy(QueueConfigurationAutoRefreshPolicy
+.class.getCanonicalName(), yarnSiteConfig));
+
 int maxAssign = conf.getInt(FairSchedulerConfiguration.MAX_ASSIGN,
 FairSchedulerConfiguration.DEFAULT_MAX_ASSIGN);
 if (maxAssign != FairSchedulerConfiguration.DEFAULT_MAX_ASSIGN) {
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java
index 55ac242..4498373 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import 
org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueDeletionPolicy;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueConfigurationAutoRefreshPolicy;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
@@ -70,6 +71,19 @@ public class TestFSYarnSiteConverter {
   }
 
   @Test
+  public void testSiteQueueConfAutoRefreshConversion() {
+converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false,
+false, false, null);
+assertTrue(yarnConvertedConfig.get(YarnConfiguration.
+RM_SCHEDULER_ENABLE_MONITORS), true);
+assertTrue("Scheduling Policies contains queue conf auto refresh",
+yarnConvertedConfig.
+ 

[hadoop] branch trunk updated: YARN-10705. Misleading DEBUG log for container assignment needs to be removed when the container is actually reserved, not assigned in FairScheduler. Contributed by Sidd

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 14a84c4  YARN-10705. Misleading DEBUG log for container assignment 
needs to be removed when the container is actually reserved, not assigned in 
FairScheduler. Contributed by Siddharth Ahuja.
14a84c4 is described below

commit 14a84c47b0efe287fb848856767b115cf5af4175
Author: Peter Bacsko 
AuthorDate: Fri Apr 23 17:38:34 2021 +0200

YARN-10705. Misleading DEBUG log for container assignment needs to be 
removed when the container is actually reserved, not assigned in FairScheduler. 
Contributed by Siddharth Ahuja.
---
 .../resourcemanager/scheduler/fair/FSLeafQueue.java  | 16 +---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 5eafa52..af366ec 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -352,9 +352,19 @@ public class FSLeafQueue extends FSQueue {
 continue;
   }
   assigned = sched.assignContainer(node);
-  if (!assigned.equals(none())) {
-LOG.debug("Assigned container in queue:{} container:{}",
-getName(), assigned);
+
+  boolean isContainerAssignedOrReserved = !assigned.equals(none());
+  boolean isContainerReserved =
+assigned.equals(FairScheduler.CONTAINER_RESERVED);
+
+  // check if an assignment or a reservation was made.
+  if (isContainerAssignedOrReserved) {
+// only log container assignment if there was an actual allocation,
+// not a reservation.
+if (!isContainerReserved && LOG.isDebugEnabled()) {
+  LOG.debug("Assigned container in queue:{} container:{}",
+  getName(), assigned);
+}
 break;
   }
 }

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10564. Support Auto Queue Creation template configurations. Contributed by Andras Gyori.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new ca9aa91  YARN-10564. Support Auto Queue Creation template 
configurations. Contributed by Andras Gyori.
ca9aa91 is described below

commit ca9aa91d102a388c6965ac0bf91e99ba8ef536b4
Author: Peter Bacsko 
AuthorDate: Thu Apr 8 12:42:48 2021 +0200

YARN-10564. Support Auto Queue Creation template configurations. 
Contributed by Andras Gyori.
---
 .../scheduler/capacity/AbstractCSQueue.java|  35 +++---
 .../capacity/AutoCreatedQueueTemplate.java | 133 +
 .../capacity/CapacitySchedulerConfiguration.java   |   2 +-
 .../scheduler/capacity/ParentQueue.java|  27 -
 .../capacity/TestAutoCreatedQueueTemplate.java | 116 ++
 .../TestCapacitySchedulerNewQueueAutoCreation.java |  80 +
 6 files changed, 375 insertions(+), 18 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 06575be..250fcc7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -18,25 +18,15 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import 
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
+import 
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -58,13 +48,13 @@ import 
org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType;
 import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
 import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AbsoluteResourceType;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AbsoluteResourceType;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerAllocationProposal;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
@@ -74,8 +64,17 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimpleC
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
+import

[hadoop] branch trunk updated: YARN-10726. Log the size of DelegationTokenRenewer event queue in case of too many pending events. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 158758c  YARN-10726. Log the size of DelegationTokenRenewer event 
queue in case of too many pending events. Contributed by Qi Zhu.
158758c is described below

commit 158758c5bf1a02d3f7829dc16948acf8389ac491
Author: Peter Bacsko 
AuthorDate: Thu Apr 1 16:09:52 2021 +0200

YARN-10726. Log the size of DelegationTokenRenewer event queue in case of 
too many pending events. Contributed by Qi Zhu.
---
 .../server/resourcemanager/security/DelegationTokenRenewer.java  | 9 +
 1 file changed, 9 insertions(+)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
index b4da08f..d8368eb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
@@ -89,6 +89,8 @@ public class DelegationTokenRenewer extends AbstractService {
   new Text("HDFS_DELEGATION_TOKEN");
   public static final String SCHEME = "hdfs";
 
+  private volatile int lastEventQueueSizeLogged = 0;
+
   // global single timer (daemon)
   private Timer renewalTimer;
   private RMContext rmContext;
@@ -228,6 +230,13 @@ public class DelegationTokenRenewer extends 
AbstractService {
 futures.put(evt, future);
   } else {
 pendingEventQueue.add(evt);
+int qSize = pendingEventQueue.size();
+if (qSize != 0 && qSize % 1000 == 0
+&& lastEventQueueSizeLogged != qSize) {
+  lastEventQueueSizeLogged = qSize;
+  LOG.info("Size of pending " +
+  "DelegationTokenRenewerEvent queue is " + qSize);
+}
   }
 } finally {
   serviceStateLock.readLock().unlock();

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-9618. NodesListManager event improvement. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 9f1655b  YARN-9618. NodesListManager event improvement. Contributed by 
Qi Zhu.
9f1655b is described below

commit 9f1655baf283075389dbe7506b4475881a47baf7
Author: Peter Bacsko 
AuthorDate: Thu Apr 1 11:39:08 2021 +0200

YARN-9618. NodesListManager event improvement. Contributed by Qi Zhu.
---
 .../server/resourcemanager/NodesListManager.java   |  8 +--
 .../rmapp/TestNodesListManager.java| 78 +++---
 2 files changed, 26 insertions(+), 60 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
index f9e1591..07d78cb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
@@ -509,12 +509,8 @@ public class NodesListManager extends CompositeService 
implements
   RMNode eventNode, RMAppNodeUpdateType appNodeUpdateType) {
 for(RMApp app : rmContext.getRMApps().values()) {
   if (!app.isAppFinalStateStored()) {
-this.rmContext
-.getDispatcher()
-.getEventHandler()
-.handle(
-new RMAppNodeUpdateEvent(app.getApplicationId(), eventNode,
-appNodeUpdateType));
+app.handle(new RMAppNodeUpdateEvent(app.getApplicationId(), eventNode,
+appNodeUpdateType));
   }
 }
   }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java
index 0df295c..35148e9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java
@@ -22,10 +22,7 @@ import static org.mockito.ArgumentMatchers.argThat;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.spy;
 
-import java.util.ArrayList;
-
 import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -52,8 +49,8 @@ import org.junit.Test;
 import org.mockito.ArgumentMatcher;
 
 public class TestNodesListManager {
-  // To hold list of application for which event was received
-  ArrayList applist = new ArrayList();
+  private boolean isRMAppEvent;
+  private boolean isNodesListEvent;
 
   @Test(timeout = 30)
   public void testNodeUsableEvent() throws Exception {
@@ -68,67 +65,32 @@ public class TestNodesListManager {
 };
 rm.start();
 MockNM nm1 = rm.registerNode("h1:1234", 28000);
-NodesListManager nodesListManager = rm.getNodesListManager();
 Resource clusterResource = Resource.newInstance(28000, 8);
 RMNode rmnode = MockNodes.newNodeInfo(1, clusterResource);
 
 // Create killing APP
-RMApp killrmApp = MockRMAppSubmitter.submitWithMemory(200, rm);
-rm.killApp(killrmApp.getApplicationId());
-rm.waitForState(killrmApp.getApplicationId(), RMAppState.KILLED);
+RMApp killRmApp = MockRMAppSubmitter.submitWithMemory(200, rm);
+rm.killApp(killRmApp.getApplicationId());
+rm.waitForState(killRmApp.getApplicationId(), RMAppState.KILLED);
 
 // Create finish APP
-RMApp finshrmApp = MockRMAppSubmitter.submitWithMemory(2000, rm);
+RMApp finshRmApp = MockRMAppSubmitter.submitWithMemory(2000, rm);
 nm1.nodeHeartbeat(true);
-RMAppAttempt attempt = finshrmApp.getCurrentAppAttempt();
+RMAppAttempt attempt = finshRmApp.getCurrentAppAttempt();
 MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
 am.registerAppAttempt();
 am.unregisterAppAttempt();
 nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE);
 rm.wa

[hadoop] branch trunk updated: YARN-10720. YARN WebAppProxyServlet should support connection timeout to prevent proxy server from hanging. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new a0deda1  YARN-10720. YARN WebAppProxyServlet should support connection 
timeout to prevent proxy server from hanging. Contributed by Qi Zhu.
a0deda1 is described below

commit a0deda1a777d8967fb8c08ac976543cda895773d
Author: Peter Bacsko 
AuthorDate: Thu Apr 1 09:21:15 2021 +0200

YARN-10720. YARN WebAppProxyServlet should support connection timeout to 
prevent proxy server from hanging. Contributed by Qi Zhu.
---
 .../apache/hadoop/yarn/conf/YarnConfiguration.java | 14 
 .../src/main/resources/yarn-default.xml| 12 
 .../yarn/server/webproxy/WebAppProxyServlet.java   | 28 ++--
 .../server/webproxy/TestWebAppProxyServlet.java| 79 +-
 4 files changed, 126 insertions(+), 7 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 2cf4a3b..2103ae7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2668,6 +2668,20 @@ public class YarnConfiguration extends Configuration {
 
   public static final String DEFAULT_RM_APPLICATION_HTTPS_POLICY = "NONE";
 
+
+  // If the proxy connection time enabled.
+  public static final String RM_PROXY_TIMEOUT_ENABLED =
+  RM_PREFIX + "proxy.timeout.enabled";
+
+  public static final boolean DEFALUT_RM_PROXY_TIMEOUT_ENABLED =
+  true;
+
+  public static final String RM_PROXY_CONNECTION_TIMEOUT =
+  RM_PREFIX + "proxy.connection.timeout";
+
+  public static final int DEFAULT_RM_PROXY_CONNECTION_TIMEOUT =
+  6;
+
   /**
* Interval of time the linux container executor should try cleaning up
* cgroups entry when cleaning up a container. This is required due to what 
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 12fc32f..c51cf8a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2574,6 +2574,18 @@
 
   
 
+  
+Enable the web proxy connection timeout, default is 
enabled.
+yarn.resourcemanager.proxy.timeout.enabled
+true
+  
+
+  
+The web proxy connection timeout.
+yarn.resourcemanager.proxy.connection.timeout
+6
+  
+
   
 
   
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
index 0b6bb65..03b7077 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
@@ -122,6 +122,9 @@ public class WebAppProxyServlet extends HttpServlet {
 }
   }
 
+  protected void setConf(YarnConfiguration conf){
+this.conf = conf;
+  }
   /**
* Default constructor
*/
@@ -230,6 +233,14 @@ public class WebAppProxyServlet extends HttpServlet {
 
 String httpsPolicy = 
conf.get(YarnConfiguration.RM_APPLICATION_HTTPS_POLICY,
 YarnConfiguration.DEFAULT_RM_APPLICATION_HTTPS_POLICY);
+
+boolean connectionTimeoutEnabled =
+conf.getBoolean(YarnConfiguration.RM_PROXY_TIMEOUT_ENABLED,
+YarnConfiguration.DEFALUT_RM_PROXY_TIMEOUT_ENABLED);
+int connectionTimeout =
+conf.getInt(YarnConfiguration.RM_PROXY_CONNECTION_TIMEOUT,
+YarnConfiguration.DEFAULT_RM_PROXY_CONNECTION_TIMEOUT);
+
 if (httpsPolicy.equals("LENIENT") || httpsPolicy.equals("STRICT")) {
   ProxyCA proxyCA = getProxyCA();
   // ProxyCA could be null when the Proxy is run outside the RM
@@ -250,10 +261,18 @@ public class WebAppProxyServlet extends HttpServlet {
 InetAddress localAddress = InetAddress.getByName(proxyHost);
 LOG.debug("local InetAddress for proxy host: {}", localAddress);
 httpClientBuilder.setDefaultRequestConfig(
-RequestConfig.custom()
-.setCircularRedirectsAllowed(t

[hadoop] branch trunk updated: YARN-10718. Fix CapacityScheduler#initScheduler log error. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new ff6ec20  YARN-10718. Fix CapacityScheduler#initScheduler log error. 
Contributed by Qi Zhu.
ff6ec20 is described below

commit ff6ec20d84560be27f287555d5bfc7dd7617585c
Author: Peter Bacsko 
AuthorDate: Wed Mar 31 10:55:14 2021 +0200

YARN-10718. Fix CapacityScheduler#initScheduler log error. Contributed by 
Qi Zhu.
---
 .../resourcemanager/scheduler/capacity/CapacityScheduler.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 70738c0..37acb60 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -393,9 +393,9 @@ public class CapacityScheduler extends
   }
 
   LOG.info("Initialized CapacityScheduler with " + "calculator="
-  + getResourceCalculator().getClass() + ", " + "minimumAllocation=<"
-  + getMinimumResourceCapability() + ">, " + "maximumAllocation=<"
-  + getMaximumResourceCapability() + ">, " + "asynchronousScheduling="
+  + getResourceCalculator().getClass() + ", " + "minimumAllocation="
+  + getMinimumResourceCapability() + ", " + "maximumAllocation="
+  + getMaximumResourceCapability() + ", " + "asynchronousScheduling="
   + scheduleAsynchronously + ", " + "asyncScheduleInterval="
   + asyncScheduleInterval + "ms" + ",multiNodePlacementEnabled="
   + multiNodePlacementEnabled);

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10674. fs2cs should generate auto-created queue deletion properties. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new ceb75e1  YARN-10674. fs2cs should generate auto-created queue deletion 
properties. Contributed by Qi Zhu.
ceb75e1 is described below

commit ceb75e1e2a81da429bca839488b9a368389055c0
Author: Peter Bacsko 
AuthorDate: Wed Mar 24 08:15:06 2021 +0100

YARN-10674. fs2cs should generate auto-created queue deletion properties. 
Contributed by Qi Zhu.
---
 .../FSConfigToCSConfigArgumentHandler.java |  29 +
 .../converter/FSConfigToCSConfigConverter.java |  15 ++-
 .../FSConfigToCSConfigConverterParams.java |  40 +++
 .../fair/converter/FSYarnSiteConverter.java|  48 +++-
 .../converter/TestFSConfigToCSConfigConverter.java |  15 +++
 .../fair/converter/TestFSYarnSiteConverter.java| 127 +++--
 6 files changed, 261 insertions(+), 13 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
index 084b67d..41e7ba5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
@@ -118,6 +118,14 @@ public class FSConfigToCSConfigArgumentHandler {
 "pc", "percentage",
 "Converts FS queue weights to percentages",
 false),
+DISABLE_PREEMPTION("disable preemption", "dp", "disable-preemption",
+"Disable the preemption with nopolicy or observeonly mode. " +
+"Preemption is enabled by default. " +
+"nopolicy removes ProportionalCapacityPreemptionPolicy from " +
+"the list of monitor policies, " +
+"observeonly sets " +
+"yarn.resourcemanager.monitor.capacity.preemption.observe_only " +
+"to true.", true),
 HELP("help", "h", "help", "Displays the list of options", false);
 
 private final String name;
@@ -251,6 +259,12 @@ public class FSConfigToCSConfigArgumentHandler {
 cliParser.getOptionValue(CliOption.CONVERSION_RULES.shortSwitch);
 String outputDir =
 cliParser.getOptionValue(CliOption.OUTPUT_DIR.shortSwitch);
+FSConfigToCSConfigConverterParams.
+PreemptionMode preemptionMode =
+FSConfigToCSConfigConverterParams.
+PreemptionMode.fromString(cliParser.
+getOptionValue(CliOption.DISABLE_PREEMPTION.shortSwitch));
+
 boolean convertPlacementRules =
 !cliParser.hasOption(
 CliOption.SKIP_PLACEMENT_RULES_CONVERSION.shortSwitch);
@@ -260,6 +274,10 @@ public class FSConfigToCSConfigArgumentHandler {
 checkFile(CliOption.CONVERSION_RULES, conversionRulesFile);
 checkDirectory(CliOption.OUTPUT_DIR, outputDir);
 checkOutputDirDoesNotContainXmls(yarnSiteXmlFile, outputDir);
+if (cliParser.hasOption(CliOption.
+DISABLE_PREEMPTION.shortSwitch)) {
+  checkDisablePreemption(preemptionMode);
+}
 
 // check mapping-rules.json if we intend to generate it
 if (!cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch) &&
@@ -281,6 +299,7 @@ public class FSConfigToCSConfigArgumentHandler {
 cliParser.hasOption(CliOption.RULES_TO_FILE.shortSwitch))
 .withUsePercentages(
 cliParser.hasOption(CliOption.CONVERT_PERCENTAGES.shortSwitch))
+.withDisablePreemption(preemptionMode)
 .build();
   }
 
@@ -383,6 +402,16 @@ public class FSConfigToCSConfigArgumentHandler {
 }
   }
 
+  private static void checkDisablePreemption(FSConfigToCSConfigConverterParams.
+  PreemptionMode preemptionMode) {
+if (preemptionMode == FSConfigToCSConfigConverterParams.
+PreemptionMode.ENABLED) {
+  throw new PreconditionException(
+  "Specified disable-preemption mode is illegal, " +
+  " use nopolicy or observeonly.");
+}
+  }
+
   private FSConfigToCSConfigConverter getConverter() {
 return new FSConfigToCSConfigConverter(ruleHandler, conversionO

[hadoop] branch trunk updated: YARN-10641. Refactor the max app related update, and fix maxApllications update error when add new queues. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new ce6bfd5  YARN-10641. Refactor the max app related update, and fix 
maxApllications update error when add new queues. Contributed by Qi Zhu.
ce6bfd5 is described below

commit ce6bfd5718a3adb124a45ce09d2ef16cfed2c1a4
Author: Peter Bacsko 
AuthorDate: Thu Mar 18 13:40:16 2021 +0100

YARN-10641. Refactor the max app related update, and fix maxApllications 
update error when add new queues. Contributed by Qi Zhu.
---
 .../scheduler/capacity/AbstractCSQueue.java| 81 --
 .../capacity/CapacitySchedulerConfiguration.java   |  7 ++
 .../scheduler/capacity/LeafQueue.java  | 32 +++--
 .../TestCapacitySchedulerNewQueueAutoCreation.java | 57 +++
 4 files changed, 118 insertions(+), 59 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 5c0f99a..06575be 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -1489,6 +1489,44 @@ public abstract class AbstractCSQueue implements CSQueue 
{
 return ret;
   }
 
+  void updateMaxAppRelatedField(CapacitySchedulerConfiguration conf,
+  LeafQueue leafQueue, String label) {
+int maxApplications = conf.getMaximumApplicationsPerQueue(queuePath);
+if (maxApplications < 0) {
+  int maxGlobalPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue();
+  if (maxGlobalPerQueueApps > 0) {
+// In absolute mode, should
+// shrink when change to corresponding label capacity.
+maxApplications = this.capacityConfigType
+!= CapacityConfigType.ABSOLUTE_RESOURCE ?
+  maxGlobalPerQueueApps :
+(int) (maxGlobalPerQueueApps * queueCapacities
+.getAbsoluteCapacity(label));
+  } else{
+maxApplications = (int) (conf.getMaximumSystemApplications()
+* queueCapacities.getAbsoluteCapacity(label));
+  }
+}
+leafQueue.setMaxApplications(maxApplications);
+
+int maxApplicationsPerUser = Math.min(maxApplications,
+(int) (maxApplications
+* (leafQueue.getUsersManager().getUserLimit() / 100.0f)
+* leafQueue.getUsersManager().getUserLimitFactor()));
+if (leafQueue.getUsersManager().getUserLimitFactor() == -1) {
+  maxApplicationsPerUser =  maxApplications;
+}
+
+leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser);
+LOG.info("LeafQueue:" + leafQueue.getQueuePath() +
+"update max app related, maxApplications="
++ maxApplications + ", maxApplicationsPerUser="
++ maxApplicationsPerUser + ", Abs Cap:" + queueCapacities
+.getAbsoluteCapacity(label) + ", Cap: " + queueCapacities
+.getCapacity(label) + ", MaxCap : " + queueCapacities
+.getMaximumCapacity(label));
+  }
+
   private void deriveCapacityFromAbsoluteConfigurations(String label,
   Resource clusterResource, ResourceCalculator rc) {
 
@@ -1522,42 +1560,6 @@ public abstract class AbstractCSQueue implements CSQueue 
{
 queueCapacities.setAbsoluteMaximumCapacity(label,
 queueCapacities.getMaximumCapacity(label) * parent.getQueueCapacities()
 .getAbsoluteMaximumCapacity(label));
-
-// Re-visit max applications for a queue based on absolute capacity if
-// needed.
-if (this instanceof LeafQueue) {
-  LeafQueue leafQueue = (LeafQueue) this;
-  CapacitySchedulerConfiguration conf = csContext.getConfiguration();
-  int maxApplications = conf.getMaximumApplicationsPerQueue(queuePath);
-  if (maxApplications < 0) {
-int maxGlobalPerQueueApps = 
conf.getGlobalMaximumApplicationsPerQueue();
-if (maxGlobalPerQueueApps > 0) {
-  maxApplications = (int) (maxGlobalPerQueueApps * queueCapacities
-  .getAbsoluteCapacity(label));
-} else{
-  maxApplications =
-  (int) (conf.getMaximumSystemApplications() * queueCapacities
-  .getAbsoluteCapacity(label));
-}
-  }
-  leafQueue.setMaxApplications(maxApplications);
-
-

[hadoop] branch trunk updated: YARN-10692. Add Node GPU Utilization and apply to NodeMetrics. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 38495af  YARN-10692. Add Node GPU Utilization and apply to 
NodeMetrics. Contributed by Qi Zhu.
38495af is described below

commit 38495af325ea16c051cbd2cf7854f54f4414e09b
Author: Peter Bacsko 
AuthorDate: Thu Mar 18 12:45:28 2021 +0100

YARN-10692. Add Node GPU Utilization and apply to NodeMetrics. Contributed 
by Qi Zhu.
---
 .../nodemanager/NodeResourceMonitorImpl.java   | 28 ++
 .../gpu/GpuNodeResourceUpdateHandler.java  | 18 +
 .../nodemanager/metrics/NodeManagerMetrics.java| 10 +
 .../TestContainerManagerRecovery.java  |  9 +++--
 .../resourceplugin/gpu/TestGpuResourcePlugin.java  | 43 ++
 .../metrics/TestNodeManagerMetrics.java|  9 -
 6 files changed, 112 insertions(+), 5 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
index c39e52e..098e7c7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
@@ -20,8 +20,11 @@ package org.apache.hadoop.yarn.server.nodemanager;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuNodeResourceUpdateHandler;
+import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin;
 import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
 import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 import org.slf4j.Logger;
@@ -46,6 +49,10 @@ public class NodeResourceMonitorImpl extends AbstractService 
implements
   /** Resource calculator. */
   private ResourceCalculatorPlugin resourceCalculatorPlugin;
 
+  /** Gpu related plugin. */
+  private GpuResourcePlugin gpuResourcePlugin;
+  private GpuNodeResourceUpdateHandler gpuNodeResourceUpdateHandler;
+
   /** Current resource utilization of the node. */
   private ResourceUtilization nodeUtilization =
   ResourceUtilization.newInstance(0, 0, 0f);
@@ -72,6 +79,18 @@ public class NodeResourceMonitorImpl extends AbstractService 
implements
 this.resourceCalculatorPlugin =
 ResourceCalculatorPlugin.getNodeResourceMonitorPlugin(conf);
 
+if (nmContext.getResourcePluginManager() != null) {
+  this.gpuResourcePlugin =
+  (GpuResourcePlugin)nmContext.getResourcePluginManager().
+  getNameToPlugins().get(ResourceInformation.GPU_URI);
+
+  if (gpuResourcePlugin != null) {
+this.gpuNodeResourceUpdateHandler =
+(GpuNodeResourceUpdateHandler)gpuResourcePlugin.
+getNodeResourceHandlerInstance();
+  }
+}
+
 LOG.info(" Using ResourceCalculatorPlugin : "
 + this.resourceCalculatorPlugin);
   }
@@ -152,6 +171,14 @@ public class NodeResourceMonitorImpl extends 
AbstractService implements
 (int) (vmem >> 20), // B -> MB
 vcores); // Used Virtual Cores
 
+float nodeGpuUtilization = 0F;
+try {
+  nodeGpuUtilization =
+  gpuNodeResourceUpdateHandler.getNodeGpuUtilization();
+} catch (Exception e) {
+  LOG.error("Get Node GPU Utilization error: " + e);
+}
+
 // Publish the node utilization metrics to node manager
 // metrics system.
 NodeManagerMetrics nmMetrics = nmContext.getNodeManagerMetrics();
@@ -159,6 +186,7 @@ public class NodeResourceMonitorImpl extends 
AbstractService implements
   nmMetrics.setNodeUsedMemGB(nodeUtilization.getPhysicalMemory());
   nmMetrics.setNodeUsedVMemGB(nodeUtilization.getVirtualMemory());
   nmMetrics.setNodeCpuUtilization(nodeUtilization.getCPU());
+  nmMetrics.setNodeGpuUtilization(nodeGpuUtilization);
 }
 
 try {
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuNodeR

[hadoop] branch trunk updated: YARN-10685. Fix typos in AbstractCSQueue. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new d7eeca4  YARN-10685. Fix typos in AbstractCSQueue. Contributed by Qi 
Zhu.
d7eeca4 is described below

commit d7eeca4d0cd32bc7e169d4c2e969d6c0d25dfd7c
Author: Peter Bacsko 
AuthorDate: Thu Mar 18 11:49:16 2021 +0100

YARN-10685. Fix typos in AbstractCSQueue. Contributed by Qi Zhu.
---
 .../resourcemanager/scheduler/capacity/AbstractCSQueue.java   | 11 ++-
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 2d85d61..5c0f99a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -114,7 +114,8 @@ public abstract class AbstractCSQueue implements CSQueue {
 
   private final boolean fullPathQueueNamingPolicy = false;
 
-  // Track capacities like 
used-capcity/abs-used-capacity/capacity/abs-capacity,
+  // Track capacities like
+  // used-capacity/abs-used-capacity/capacity/abs-capacity,
   // etc.
   QueueCapacities queueCapacities;
 
@@ -962,7 +963,7 @@ public abstract class AbstractCSQueue implements CSQueue {
 if (defaultAppLifetime < 0) {
   // If default lifetime was not set at this level but was set somewhere in
   // the parent's hierarchy, set default lifetime to parent queue's default
-  // only if parent queue's lifetime is less than current queueu's max
+  // only if parent queue's lifetime is less than current queue's max
   // lifetime. Otherwise, use current queue's max lifetime value for its
   // default lifetime.
   if (defaultAppLifetimeWasSpecifiedInConfig) {
@@ -1012,7 +1013,7 @@ public abstract class AbstractCSQueue implements CSQueue {
 
 // At this point, the master preemption switch is enabled down to this
 // queue's level. Determine whether or not intra-queue preemption is 
enabled
-// down to this queu's level and return that value.
+// down to this queue's level and return that value.
 return configuration.getIntraQueuePreemptionDisabled(q.getQueuePath(),
 parentQ.getIntraQueuePreemptionDisabledInHierarchy());
   }
@@ -1065,7 +1066,7 @@ public abstract class AbstractCSQueue implements CSQueue {
   // Doing this because: for non-exclusive allocation, we make sure there's
   // idle resource on the partition, to avoid wastage, such resource will 
be
   // leveraged as much as we can, and preemption policy will reclaim it 
back
-  // when partitoned-resource-request comes back.
+  // when partitioned-resource-request comes back.
   Resource currentLimitResource = getCurrentLimitResource(nodePartition,
   clusterResource, currentResourceLimits, schedulingMode);
 
@@ -1115,7 +1116,7 @@ public abstract class AbstractCSQueue implements CSQueue {
 // Can not assign to this queue
 if (LOG.isDebugEnabled()) {
   LOG.debug("Failed to assign to queue: " + getQueuePath()
-  + " nodePatrition: " + nodePartition
+  + " nodePartition: " + nodePartition
   + ", usedResources: " + queueUsage.getUsed(nodePartition)
   + ", clusterResources: " + clusterResource
   + ", reservedResources: " + resourceCouldBeUnreserved


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10497. Fix an issue in CapacityScheduler which fails to delete queues. Contributed by Wangda Tan and Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 3e58d56  YARN-10497. Fix an issue in CapacityScheduler which fails to 
delete queues. Contributed by Wangda Tan and Qi Zhu.
3e58d56 is described below

commit 3e58d5611d2426f576e667ceccc5c8f64c9699e2
Author: Peter Bacsko 
AuthorDate: Wed Mar 17 13:38:20 2021 +0100

YARN-10497. Fix an issue in CapacityScheduler which fails to delete queues. 
Contributed by Wangda Tan and Qi Zhu.
---
 .../conf/MutableCSConfigurationProvider.java   | 12 ++--
 .../conf/TestMutableCSConfigurationProvider.java   | 33 ++
 2 files changed, 43 insertions(+), 2 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 5f262be..b74414b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -68,10 +68,18 @@ public class MutableCSConfigurationProvider implements 
CSConfigurationProvider,
 this.rmContext = rmContext;
   }
 
+  // Unit test can overwrite this method
+  protected Configuration getInitSchedulerConfig() {
+Configuration initialSchedConf = new Configuration(false);
+initialSchedConf.
+addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
+return initialSchedConf;
+  }
+
   @Override
   public void init(Configuration config) throws IOException {
 this.confStore = YarnConfigurationStoreFactory.getStore(config);
-Configuration initialSchedConf = new Configuration(false);
+Configuration initialSchedConf = getInitSchedulerConfig();
 initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
 this.schedConf = new Configuration(false);
 // We need to explicitly set the key-values in schedConf, otherwise
@@ -231,7 +239,7 @@ public class MutableCSConfigurationProvider implements 
CSConfigurationProvider,
 String childQueuesKey = CapacitySchedulerConfiguration.PREFIX +
 parentQueue + CapacitySchedulerConfiguration.DOT +
 CapacitySchedulerConfiguration.QUEUES;
-return new ArrayList<>(conf.getStringCollection(childQueuesKey));
+return new ArrayList<>(conf.getTrimmedStringCollection(childQueuesKey));
   }
 
   private Map constructKeyValueConfUpdate(
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 0c9a312..aec2c21 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -217,6 +217,39 @@ public class TestMutableCSConfigurationProvider {
 
   }
 
+  @Test
+  public void testAddRemoveQueueWithSpacesInConfig() throws Exception {
+CapacitySchedulerConfiguration csConf =
+new CapacitySchedulerConfiguration();
+csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+new String[] {" a   , b, c" });
+
+final String a = CapacitySchedulerConfiguration.ROOT + ".a";
+final String b = CapacitySchedulerConfiguration.ROOT + ".b";
+final String c = CapacitySchedulerConfiguration.ROOT + ".c";
+csConf.setCapacity(a, 0);
+csConf.setCapacity(b, 50);
+csConf.setCapacity(c, 50);
+
+confProvider = new MutableCSConfigurationProvider(rmContext) {
+  @Override
+  protected Configuration getInitSchedulerConfig() {
+return csConf;
+  }
+};
+
+Configuration conf = new Configuration();
+conf.set(Y

[hadoop] branch trunk updated: YARN-10686. Fix TestCapacitySchedulerAutoQueueCreation#testAutoQueueCreationFailsForEmptyPathWithAQCAndWeightMode. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 73394fab YARN-10686. Fix 
TestCapacitySchedulerAutoQueueCreation#testAutoQueueCreationFailsForEmptyPathWithAQCAndWeightMode.
 Contributed by Qi Zhu.
73394fab is described below

commit 73394fabc7a6e4b3cfb28b13dedc3433f2e6cc49
Author: Peter Bacsko 
AuthorDate: Tue Mar 16 16:09:12 2021 +0100

YARN-10686. Fix 
TestCapacitySchedulerAutoQueueCreation#testAutoQueueCreationFailsForEmptyPathWithAQCAndWeightMode.
 Contributed by Qi Zhu.
---
 .../scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
index be41a33..ece13c9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
@@ -571,7 +571,7 @@ public class TestCapacitySchedulerAutoQueueCreation
 
   ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
   SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(
-  appId, "user", "root.");
+  appId, "user", USER0);
   newCS.handle(addAppEvent);
 
   RMAppEvent event = new RMAppEvent(appId, RMAppEventType.APP_REJECTED,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10682. The scheduler monitor policies conf should trim values separated by comma. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new b80588b  YARN-10682. The scheduler monitor policies conf should trim 
values separated by comma. Contributed by Qi Zhu.
b80588b is described below

commit b80588b6883b44f24838d1525c9b267186586042
Author: Peter Bacsko 
AuthorDate: Tue Mar 16 15:23:27 2021 +0100

YARN-10682. The scheduler monitor policies conf should trim values 
separated by comma. Contributed by Qi Zhu.
---
 .../yarn/server/resourcemanager/monitor/SchedulingMonitorManager.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitorManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitorManager.java
index 5418e8b..f3c10ea 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitorManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitorManager.java
@@ -61,7 +61,7 @@ public class SchedulingMonitorManager {
 }
 
 // When monitor is enabled, loading policies
-String[] configuredPolicies = conf.getStrings(
+String[] configuredPolicies = conf.getTrimmedStrings(
 YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES);
 if (configuredPolicies == null || configuredPolicies.length == 0) {
   return;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] 02/02: YARN-9615. Add dispatcher metrics to RM. Contributed by Jonathan Hung and Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit d5e035dbe1b93e6d17c0ceef68f67658a2ffbb11
Author: Peter Bacsko 
AuthorDate: Tue Mar 9 14:33:14 2021 +0100

YARN-9615. Add dispatcher metrics to RM. Contributed by Jonathan Hung and 
Qi Zhu.
---
 .../apache/hadoop/yarn/event/AsyncDispatcher.java  |  26 ++-
 .../apache/hadoop/yarn/event/EventDispatcher.java  |  20 ++-
 .../yarn/metrics/DisableEventTypeMetrics.java  |  42 +
 .../hadoop/yarn/metrics/EventTypeMetrics.java  |  32 
 .../yarn/metrics/GenericEventTypeMetrics.java  | 160 ++
 .../hadoop/yarn/event/TestAsyncDispatcher.java | 182 -
 .../GenericEventTypeMetricsManager.java|  42 +
 .../server/resourcemanager/ResourceManager.java|  18 +-
 .../resourcemanager/TestResourceManager.java   |   2 +
 .../scheduler/TestSchedulerHealth.java |   2 +
 .../scheduler/capacity/TestCapacityScheduler.java  |   1 +
 11 files changed, 521 insertions(+), 6 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index f9deab0..667515d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -26,6 +26,9 @@ import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
+import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.MonotonicClock;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.Marker;
@@ -85,6 +88,11 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   protected final Map, EventHandler> eventDispatchers;
   private boolean exitOnDispatchException = true;
 
+  private Map,
+  EventTypeMetrics> eventTypeMetricsMap;
+
+  private Clock clock = new MonotonicClock();
+
   /**
* The thread name for dispatcher.
*/
@@ -98,6 +106,8 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
 super("Dispatcher");
 this.eventQueue = eventQueue;
 this.eventDispatchers = new HashMap, EventHandler>();
+this.eventTypeMetricsMap = new HashMap,
+EventTypeMetrics>();
   }
 
   /**
@@ -135,7 +145,16 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
 return;
   }
   if (event != null) {
-dispatch(event);
+if (eventTypeMetricsMap.
+get(event.getType().getDeclaringClass()) != null) {
+  long startTime = clock.getTime();
+  dispatch(event);
+  eventTypeMetricsMap.get(event.getType().getDeclaringClass())
+  .increment(event.getType(),
+  clock.getTime() - startTime);
+} else {
+  dispatch(event);
+}
 if (printTrigger) {
   //Log the latest dispatch event type
   // may cause the too many events queued
@@ -369,4 +388,9 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   protected boolean isStopped() {
 return stopped;
   }
+
+  public void addMetrics(EventTypeMetrics metrics,
+  Class eventClass) {
+eventTypeMetricsMap.put(eventClass, metrics);
+  }
 }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
index 0969e99..f51bae6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
@@ -19,6 +19,9 @@
 package org.apache.hadoop.yarn.event;
 
 import 
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.MonotonicClock;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.Marker;
@@ -47,12 +50,15 @@ public class EventDispatcher extends
   private final Thread eventProcessor;
   private volatile boolean stopped = false;
   private boolean shouldExitOnError = true;
+  private EventTypeMetrics metrics;
 
   private static final Logger LOG =
   L

[hadoop] 01/02: Revert "YARN-9615. Add dispatcher metrics to RM. Contributed by Qi Zhu."

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit 3851994cd69516f82e4494774f9eb4d8c1e8f40d
Author: Peter Bacsko 
AuthorDate: Tue Mar 9 14:32:02 2021 +0100

Revert "YARN-9615. Add dispatcher metrics to RM. Contributed by Qi Zhu."

This reverts commit 369f75b7a7f62ba454cc2a102ded6c2c947e80e5.
---
 .../apache/hadoop/yarn/event/AsyncDispatcher.java  |  26 +--
 .../apache/hadoop/yarn/event/EventDispatcher.java  |  20 +--
 .../yarn/metrics/DisableEventTypeMetrics.java  |  42 -
 .../hadoop/yarn/metrics/EventTypeMetrics.java  |  32 
 .../yarn/metrics/GenericEventTypeMetrics.java  | 160 --
 .../hadoop/yarn/event/TestAsyncDispatcher.java | 182 +
 .../GenericEventTypeMetricsManager.java|  42 -
 .../server/resourcemanager/ResourceManager.java|  18 +-
 .../resourcemanager/TestResourceManager.java   |   2 -
 .../scheduler/TestSchedulerHealth.java |   2 -
 .../scheduler/capacity/TestCapacityScheduler.java  |   1 -
 11 files changed, 6 insertions(+), 521 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 667515d..f9deab0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -26,9 +26,6 @@ import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
-import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
-import org.apache.hadoop.yarn.util.Clock;
-import org.apache.hadoop.yarn.util.MonotonicClock;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.Marker;
@@ -88,11 +85,6 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   protected final Map, EventHandler> eventDispatchers;
   private boolean exitOnDispatchException = true;
 
-  private Map,
-  EventTypeMetrics> eventTypeMetricsMap;
-
-  private Clock clock = new MonotonicClock();
-
   /**
* The thread name for dispatcher.
*/
@@ -106,8 +98,6 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
 super("Dispatcher");
 this.eventQueue = eventQueue;
 this.eventDispatchers = new HashMap, EventHandler>();
-this.eventTypeMetricsMap = new HashMap,
-EventTypeMetrics>();
   }
 
   /**
@@ -145,16 +135,7 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
 return;
   }
   if (event != null) {
-if (eventTypeMetricsMap.
-get(event.getType().getDeclaringClass()) != null) {
-  long startTime = clock.getTime();
-  dispatch(event);
-  eventTypeMetricsMap.get(event.getType().getDeclaringClass())
-  .increment(event.getType(),
-  clock.getTime() - startTime);
-} else {
-  dispatch(event);
-}
+dispatch(event);
 if (printTrigger) {
   //Log the latest dispatch event type
   // may cause the too many events queued
@@ -388,9 +369,4 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   protected boolean isStopped() {
 return stopped;
   }
-
-  public void addMetrics(EventTypeMetrics metrics,
-  Class eventClass) {
-eventTypeMetricsMap.put(eventClass, metrics);
-  }
 }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
index f51bae6..0969e99 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
@@ -19,9 +19,6 @@
 package org.apache.hadoop.yarn.event;
 
 import 
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
-import org.apache.hadoop.yarn.util.Clock;
-import org.apache.hadoop.yarn.util.MonotonicClock;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.Marker;
@@ -50,15 +47,12 @@ public class EventDispatcher extends
   private final Thread eventProcessor;
   private volatile boolean stopped = false;
   private boolean shouldExitOnError = true;
-  private EventTypeMetrics me

[hadoop] branch trunk updated (369f75b -> d5e035d)

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git.


from 369f75b  YARN-9615. Add dispatcher metrics to RM. Contributed by Qi 
Zhu.
 new 3851994  Revert "YARN-9615. Add dispatcher metrics to RM. Contributed 
by Qi Zhu."
 new d5e035d  YARN-9615. Add dispatcher metrics to RM. Contributed by 
Jonathan Hung and Qi Zhu.

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-9615. Add dispatcher metrics to RM. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 369f75b  YARN-9615. Add dispatcher metrics to RM. Contributed by Qi 
Zhu.
369f75b is described below

commit 369f75b7a7f62ba454cc2a102ded6c2c947e80e5
Author: Peter Bacsko 
AuthorDate: Tue Mar 9 14:28:23 2021 +0100

YARN-9615. Add dispatcher metrics to RM. Contributed by Qi Zhu.
---
 .../apache/hadoop/yarn/event/AsyncDispatcher.java  |  26 ++-
 .../apache/hadoop/yarn/event/EventDispatcher.java  |  20 ++-
 .../yarn/metrics/DisableEventTypeMetrics.java  |  42 +
 .../hadoop/yarn/metrics/EventTypeMetrics.java  |  32 
 .../yarn/metrics/GenericEventTypeMetrics.java  | 160 ++
 .../hadoop/yarn/event/TestAsyncDispatcher.java | 182 -
 .../GenericEventTypeMetricsManager.java|  42 +
 .../server/resourcemanager/ResourceManager.java|  18 +-
 .../resourcemanager/TestResourceManager.java   |   2 +
 .../scheduler/TestSchedulerHealth.java |   2 +
 .../scheduler/capacity/TestCapacityScheduler.java  |   1 +
 11 files changed, 521 insertions(+), 6 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index f9deab0..667515d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -26,6 +26,9 @@ import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
+import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.MonotonicClock;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.Marker;
@@ -85,6 +88,11 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   protected final Map, EventHandler> eventDispatchers;
   private boolean exitOnDispatchException = true;
 
+  private Map,
+  EventTypeMetrics> eventTypeMetricsMap;
+
+  private Clock clock = new MonotonicClock();
+
   /**
* The thread name for dispatcher.
*/
@@ -98,6 +106,8 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
 super("Dispatcher");
 this.eventQueue = eventQueue;
 this.eventDispatchers = new HashMap, EventHandler>();
+this.eventTypeMetricsMap = new HashMap,
+EventTypeMetrics>();
   }
 
   /**
@@ -135,7 +145,16 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
 return;
   }
   if (event != null) {
-dispatch(event);
+if (eventTypeMetricsMap.
+get(event.getType().getDeclaringClass()) != null) {
+  long startTime = clock.getTime();
+  dispatch(event);
+  eventTypeMetricsMap.get(event.getType().getDeclaringClass())
+  .increment(event.getType(),
+  clock.getTime() - startTime);
+} else {
+  dispatch(event);
+}
 if (printTrigger) {
   //Log the latest dispatch event type
   // may cause the too many events queued
@@ -369,4 +388,9 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   protected boolean isStopped() {
 return stopped;
   }
+
+  public void addMetrics(EventTypeMetrics metrics,
+  Class eventClass) {
+eventTypeMetricsMap.put(eventClass, metrics);
+  }
 }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
index 0969e99..f51bae6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/EventDispatcher.java
@@ -19,6 +19,9 @@
 package org.apache.hadoop.yarn.event;
 
 import 
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.MonotonicClock;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.Marker;
@@ -47,12 +50,15 @@ public class EventDispatcher extends
   private final Thread eventProcessor;
   private volatile b

[hadoop] branch trunk updated: YARN-10679. Better logging of uncaught exceptions throughout SLS. Contributed by Szilard Nemeth.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new c3aa413  YARN-10679. Better logging of uncaught exceptions throughout 
SLS. Contributed by Szilard Nemeth.
c3aa413 is described below

commit c3aa413ee3f8b24fb216d7136e71d3b0b770
Author: Peter Bacsko 
AuthorDate: Tue Mar 9 14:02:12 2021 +0100

YARN-10679. Better logging of uncaught exceptions throughout SLS. 
Contributed by Szilard Nemeth.
---
 .../apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java| 7 +++
 .../org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java| 6 +++---
 .../java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java | 6 +-
 .../src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java   | 8 ++--
 4 files changed, 17 insertions(+), 10 deletions(-)

diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index a318204..e0cb151 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -63,7 +63,6 @@ import com.codahale.metrics.Timer;
 @Unstable
 public class SLSCapacityScheduler extends CapacityScheduler implements
 SchedulerWrapper,Configurable {
-
   private Configuration conf;
  
   private Map appQueueMap =
@@ -99,7 +98,7 @@ public class SLSCapacityScheduler extends CapacityScheduler 
implements
 CapacityScheduler.class);
 schedulerMetrics.init(this, conf);
   } catch (Exception e) {
-e.printStackTrace();
+LOG.error("Caught exception while initializing schedulerMetrics", e);
   }
 }
   }
@@ -129,7 +128,7 @@ public class SLSCapacityScheduler extends CapacityScheduler 
implements
   updateQueueWithAllocateRequest(allocation, attemptId,
   resourceRequests, containerIds);
 } catch (IOException e) {
-  e.printStackTrace();
+  LOG.error("Caught exception while executing finally block", e);
 }
   }
 } else {
@@ -376,7 +375,7 @@ public class SLSCapacityScheduler extends CapacityScheduler 
implements
 schedulerMetrics.tearDown();
   }
 } catch (Exception e) {
-  e.printStackTrace();
+  LOG.error("Caught exception while stopping service", e);
 }
 super.serviceStop();
   }
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
index 7298c05..84549bc 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -92,7 +92,7 @@ public class SLSFairScheduler extends FairScheduler
 FairScheduler.class);
 schedulerMetrics.init(this, conf);
   } catch (Exception e) {
-e.printStackTrace();
+LOG.error("Caught exception while initializing schedulerMetrics", e);
   }
 }
   }
@@ -122,7 +122,7 @@ public class SLSFairScheduler extends FairScheduler
   updateQueueWithAllocateRequest(allocation, attemptId,
   resourceRequests, containerIds);
 } catch (IOException e) {
-  e.printStackTrace();
+  LOG.error("Caught exception while executing finally block", e);
 }
   }
 } else {
@@ -332,7 +332,7 @@ public class SLSFairScheduler extends FairScheduler
 schedulerMetrics.tearDown();
   }
 } catch (Exception e) {
-  e.printStackTrace();
+  LOG.error("Caught exception while stopping service", e);
 }
 super.serviceStop();
   }
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
index 19cfe88..1678455 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java
@@ -26,10 +26,14 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @Private
 @Unstable
 public class TaskRunner {
+  private static final Logger LOG = LoggerFactory.getLogger(TaskRunner.class);
+
   @Private
   @Unsta

[hadoop] branch trunk updated: YARN-10681. Fix assertion failure message in BaseSLSRunnerTest. Contributed by Szilard Nemeth.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 099f58f  YARN-10681. Fix assertion failure message in 
BaseSLSRunnerTest. Contributed by Szilard Nemeth.
099f58f is described below

commit 099f58f8f41d6439643e625794c66ad932bae17b
Author: Peter Bacsko 
AuthorDate: Tue Mar 9 13:22:48 2021 +0100

YARN-10681. Fix assertion failure message in BaseSLSRunnerTest. Contributed 
by Szilard Nemeth.
---
 .../src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
 
b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
index bfbd592..1bb4710 100644
--- 
a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
+++ 
b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/BaseSLSRunnerTest.java
@@ -126,7 +126,7 @@ public abstract class BaseSLSRunnerTest {
 
   if (!exceptionList.isEmpty()) {
 sls.stop();
-Assert.fail("TestSLSRunner catched exception from child thread "
+Assert.fail("TestSLSRunner caught exception from child thread "
 + "(TaskRunner.TaskDefinition): " + exceptionList);
 break;
   }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10677. Logger of SLSFairScheduler is provided with the wrong class. Contributed by Szilard Nemeth.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 7f522c9  YARN-10677. Logger of SLSFairScheduler is provided with the 
wrong class. Contributed by Szilard Nemeth.
7f522c9 is described below

commit 7f522c92fabc142c1b2e694e85a731a815405504
Author: Peter Bacsko 
AuthorDate: Tue Mar 9 12:53:32 2021 +0100

YARN-10677. Logger of SLSFairScheduler is provided with the wrong class. 
Contributed by Szilard Nemeth.
---
 .../java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java| 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
index c832436..7298c05 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -66,7 +66,8 @@ public class SLSFairScheduler extends FairScheduler
   new ConcurrentHashMap<>();
 
   // logger
-  private static final Logger LOG = 
LoggerFactory.getLogger(SLSCapacityScheduler.class);
+  private static final Logger LOG =
+  LoggerFactory.getLogger(SLSFairScheduler.class);
 
   public SchedulerMetrics getSchedulerMetrics() {
 return schedulerMetrics;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10678. Try blocks without catch blocks in SLS scheduler classes can swallow other exceptions. Contributed by Szilard Nemeth.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new ea90cd3  YARN-10678. Try blocks without catch blocks in SLS scheduler 
classes can swallow other exceptions. Contributed by Szilard Nemeth.
ea90cd3 is described below

commit ea90cd3556f9f4c53f657266dfbfcc7bd4b031ce
Author: Peter Bacsko 
AuthorDate: Tue Mar 9 12:03:53 2021 +0100

YARN-10678. Try blocks without catch blocks in SLS scheduler classes can 
swallow other exceptions. Contributed by Szilard Nemeth.
---
 .../org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java | 3 +++
 .../java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java| 3 +++
 2 files changed, 6 insertions(+)

diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
index f34a3d3..a318204 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
@@ -119,6 +119,9 @@ public class SLSCapacityScheduler extends CapacityScheduler 
implements
 containerIds, strings,
 strings2, updateRequests);
 return allocation;
+  } catch (Exception e) {
+LOG.error("Caught exception from allocate", e);
+throw e;
   } finally {
 context.stop();
 schedulerMetrics.increaseSchedulerAllocationCounter();
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
index 09ef2cf..c832436 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
@@ -111,6 +111,9 @@ public class SLSFairScheduler extends FairScheduler
 schedulingRequests, containerIds,
 blacklistAdditions, blacklistRemovals, updateRequests);
 return allocation;
+  } catch (Exception e) {
+LOG.error("Caught exception from allocate", e);
+throw e;
   } finally {
 context.stop();
 schedulerMetrics.increaseSchedulerAllocationCounter();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10675. Consolidate YARN-10672 and YARN-10447. Contributed by Szilard Nemeth.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 20416bc  YARN-10675. Consolidate YARN-10672 and YARN-10447. 
Contributed by Szilard Nemeth.
20416bc is described below

commit 20416bc62dce85c2c1aecbc36b8373993a3c55c3
Author: Peter Bacsko 
AuthorDate: Tue Mar 9 11:34:37 2021 +0100

YARN-10675. Consolidate YARN-10672 and YARN-10447. Contributed by Szilard 
Nemeth.
---
 .../scheduler/capacity/CapacityScheduler.java| 20 +++-
 .../scheduler/capacity/TestLeafQueue.java|  7 +--
 2 files changed, 8 insertions(+), 19 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 467dacb..d259c93 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -249,8 +249,6 @@ public class CapacityScheduler extends
 
   private CSMaxRunningAppsEnforcer maxRunningEnforcer;
 
-  private boolean activitiesManagerEnabled = true;
-
   public CapacityScheduler() {
 super(CapacityScheduler.class.getName());
 this.maxRunningEnforcer = new CSMaxRunningAppsEnforcer(this);
@@ -351,9 +349,7 @@ public class CapacityScheduler extends
   this.workflowPriorityMappingsMgr = new WorkflowPriorityMappingsManager();
 
   this.activitiesManager = new ActivitiesManager(rmContext);
-  if (activitiesManagerEnabled) {
-activitiesManager.init(conf);
-  }
+  activitiesManager.init(conf);
   initializeQueues(this.conf);
   this.isLazyPreemptionEnabled = conf.getLazyPreemptionEnabled();
 
@@ -411,9 +407,7 @@ public class CapacityScheduler extends
   private void startSchedulerThreads() {
 writeLock.lock();
 try {
-  if (activitiesManagerEnabled) {
-activitiesManager.start();
-  }
+  activitiesManager.start();
   if (scheduleAsynchronously) {
 Preconditions.checkNotNull(asyncSchedulerThreads,
 "asyncSchedulerThreads is null");
@@ -447,9 +441,7 @@ public class CapacityScheduler extends
   public void serviceStop() throws Exception {
 writeLock.lock();
 try {
-  if (activitiesManagerEnabled) {
-this.activitiesManager.stop();
-  }
+  this.activitiesManager.stop();
   if (scheduleAsynchronously && asyncSchedulerThreads != null) {
 for (Thread t : asyncSchedulerThreads) {
   t.interrupt();
@@ -3480,7 +3472,6 @@ public class CapacityScheduler extends
 this.maxRunningEnforcer = enforcer;
   }
 
-
   /**
* Returning true as capacity scheduler supports placement constraints.
*/
@@ -3490,11 +3481,6 @@ public class CapacityScheduler extends
   }
 
   @VisibleForTesting
-  public void setActivitiesManagerEnabled(boolean enabled) {
-this.activitiesManagerEnabled = enabled;
-  }
-
-  @VisibleForTesting
   public void setQueueManager(CapacitySchedulerQueueManager qm) {
 this.queueManager = qm;
   }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 6262d85..4b02310 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -162,9 +162,13 @@ public class TestLeafQueue {
   private void setUpInternal(ResourceCalculator rC, boolean withNodeLabels)
   throws Exception {
 CapacityScheduler spyCs = new CapacityScheduler();
-spyCs.setActivitiesManagerEnabled(false);
 queues = new CSQueueStore();
 cs = spy(spyCs);
+
+//All stub calls on the spy object of the 'cs' field should happen
+//before cs.start() is invoked. See YARN-10672 for more details.
+when(cs.getNumClusterNod

[hadoop] branch trunk updated: YARN-10676. Improve code quality in TestTimelineAuthenticationFilterForV1. Contributed by Szilard Nemeth.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new e472ee2  YARN-10676. Improve code quality in 
TestTimelineAuthenticationFilterForV1. Contributed by Szilard Nemeth.
e472ee2 is described below

commit e472ee2aa580f9af3d95ff0267eef69ef5e3a7d5
Author: Peter Bacsko 
AuthorDate: Tue Mar 9 11:28:24 2021 +0100

YARN-10676. Improve code quality in TestTimelineAuthenticationFilterForV1. 
Contributed by Szilard Nemeth.
---
 .../TestTimelineAuthenticationFilterForV1.java | 98 ++
 1 file changed, 62 insertions(+), 36 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
index 0e1310e..cf2db2f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.yarn.server.timeline.security;
 
-import static org.junit.Assert.assertTrue;
-
 import java.io.File;
 import java.security.PrivilegedExceptionAction;
 import java.util.Arrays;
@@ -49,12 +47,16 @@ import 
org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistor
 import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
 import static 
org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
+import static org.junit.Assert.fail;
+
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Test cases for authentication via TimelineAuthenticationFilter while
@@ -62,17 +64,20 @@ import org.junit.runners.Parameterized;
  */
 @RunWith(Parameterized.class)
 public class TestTimelineAuthenticationFilterForV1 {
+  private static final Logger LOG =
+  LoggerFactory.getLogger(TestTimelineAuthenticationFilterForV1.class);
 
   private static final String FOO_USER = "foo";
   private static final String BAR_USER = "bar";
   private static final String HTTP_USER = "HTTP";
+  private static final String PRINCIPAL = HTTP_USER + "/localhost";
 
   private static final File TEST_ROOT_DIR = new File(
   System.getProperty("test.build.dir", "target/test-dir"),
   TestTimelineAuthenticationFilterForV1.class.getName() + "-root");
-  private static File httpSpnegoKeytabFile = new File(
+  private static final File httpSpnegoKeytabFile = new File(
   KerberosTestUtils.getKeytabFile());
-  private static String httpSpnegoPrincipal =
+  private static final String httpSpnegoPrincipal =
   KerberosTestUtils.getServerPrincipal();
   private static final String BASEDIR =
   System.getProperty("test.build.dir", "target/test-dir") + "/"
@@ -100,16 +105,16 @@ public class TestTimelineAuthenticationFilterForV1 {
   testMiniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR);
   testMiniKDC.start();
   testMiniKDC.createPrincipal(
-  httpSpnegoKeytabFile, HTTP_USER + "/localhost");
+  httpSpnegoKeytabFile, PRINCIPAL);
 } catch (Exception e) {
-  assertTrue("Couldn't setup MiniKDC", false);
+  LOG.error("Failed to setup MiniKDC", e);
+  fail("Couldn't setup MiniKDC");
 }
 
 try {
   testTimelineServer = new ApplicationHistoryServer();
   conf = new Configuration(false);
-  conf.setStrings(TIMELINE_HTTP_AUTH_PREFIX + "type",
-  "kerberos");
+  conf.setStrings(TIMELINE_HTTP_AUTH_PREFIX + "type", "kerberos");
   conf.set(TIMELINE_HTTP_AUTH_PREFIX +
   KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal);
   conf.set(TIMELINE_HTTP_AUTH_PREFIX +
@@ -150,8 +155,8 @@ public class TestTimelineAuthenticationFilterForV1 {
   testTimelineServer.init(conf);
   testTimelineServer.start();
 } catch (Exception e) {
-  e.printStackTrace();
-  assertTrue("Couldn't setup TimelineServer", fal

[hadoop] branch branch-3.2 updated: YARN-10672. All testcases in TestReservations are flaky. Contributed by Szilard Nemeth.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 0b8bfc5  YARN-10672. All testcases in TestReservations are flaky. 
Contributed by Szilard Nemeth.
0b8bfc5 is described below

commit 0b8bfc50c6e759a5693c409cb760b6ad4b316f11
Author: Peter Bacsko 
AuthorDate: Mon Mar 8 14:32:13 2021 +0100

YARN-10672. All testcases in TestReservations are flaky. Contributed by 
Szilard Nemeth.
---
 .../server/resourcemanager/scheduler/capacity/TestReservations.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 625b1e7..53c2318 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -106,7 +106,6 @@ public class TestReservations {
 CapacityScheduler spyCs = new CapacityScheduler();
 cs = spy(spyCs);
 rmContext = TestUtils.getMockRMContext();
-
   }
 
   private void setup(CapacitySchedulerConfiguration csConf) throws Exception {
@@ -115,6 +114,9 @@ public class TestReservations {
 
   private void setup(CapacitySchedulerConfiguration csConf,
   boolean addUserLimits) throws Exception {
+//All stub calls on the spy object of the 'cs' field should happen
+//before cs.start() is invoked. See YARN-10672 for more details.
+when(cs.getNumClusterNodes()).thenReturn(3);
 
 csConf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, 
true);
 final String newRoot = "root" + System.currentTimeMillis();
@@ -157,8 +159,6 @@ public class TestReservations {
 cs.setRMContext(spyRMContext);
 cs.init(csConf);
 cs.start();
-
-when(cs.getNumClusterNodes()).thenReturn(3);
   }
 
   private static final String A = "a";


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch branch-3.2 updated: YARN-10642. Race condition: AsyncDispatcher can get stuck by the changes introduced in YARN-8995. Contributed by zhengchenyu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 0d032d7  YARN-10642. Race condition: AsyncDispatcher can get stuck by 
the changes introduced in YARN-8995. Contributed by zhengchenyu.
0d032d7 is described below

commit 0d032d7b6856903d754a588ca7e46184f869d4c4
Author: Peter Bacsko 
AuthorDate: Mon Mar 8 14:22:14 2021 +0100

YARN-10642. Race condition: AsyncDispatcher can get stuck by the changes 
introduced in YARN-8995. Contributed by zhengchenyu.
---
 .../apache/hadoop/yarn/event/AsyncDispatcher.java  | 19 ---
 .../hadoop/yarn/event/TestAsyncDispatcher.java | 58 +-
 2 files changed, 68 insertions(+), 9 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 81448c5..acfba27 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -20,11 +20,11 @@ package org.apache.hadoop.yarn.event;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.stream.Collectors;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -265,11 +265,16 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   }
 
   class GenericEventHandler implements EventHandler {
-private void printEventQueueDetails(BlockingQueue queue) {
-  Map counterMap = eventQueue.stream().
-  collect(Collectors.
-  groupingBy(e -> e.getType(), Collectors.counting())
-  );
+private void printEventQueueDetails() {
+  Iterator iterator = eventQueue.iterator();
+  Map counterMap = new HashMap<>();
+  while (iterator.hasNext()) {
+Enum eventType = iterator.next().getType();
+if (!counterMap.containsKey(eventType)) {
+  counterMap.put(eventType, 0L);
+}
+counterMap.put(eventType, counterMap.get(eventType) + 1);
+  }
   for (Map.Entry entry : counterMap.entrySet()) {
 long num = entry.getValue();
 LOG.info("Event type: " + entry.getKey()
@@ -292,7 +297,7 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   if (qSize != 0 && qSize % detailsInterval == 0
   && lastEventDetailsQueueSizeLogged != qSize) {
 lastEventDetailsQueueSizeLogged = qSize;
-printEventQueueDetails(eventQueue);
+printEventQueueDetails();
 printTrigger = true;
   }
   int remCapacity = eventQueue.remainingCapacity();
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
index 53e2d05..88855fc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
@@ -97,12 +97,23 @@ public class TestAsyncDispatcher {
   }
 
   private static class TestHandler implements EventHandler {
+
+private long sleepTime = 1500;
+
+TestHandler() {
+}
+
+TestHandler(long sleepTime) {
+  this.sleepTime = sleepTime;
+}
+
 @Override
 public void handle(Event event) {
   try {
 // As long as 1 events queued
-Thread.sleep(1500);
-  } catch (InterruptedException e) {}
+Thread.sleep(this.sleepTime);
+  } catch (InterruptedException e) {
+  }
 }
   }
 
@@ -170,11 +181,54 @@ public class TestAsyncDispatcher {
   //Make sure more than one event to take
   verify(log, atLeastOnce()).
   info("Latest dispatch event type: TestEventType");
+} finally {
+  //... restore logger object
+  logger.set(null, oldLog);
   dispatcher.stop();
+}
+  }
+
+  //Test print dispatcher details when the blocking queue is heavy
+  @Test(timeout = 6)
+  public void testPrintDispatcherEventDetailsAvoidDeadLoop() throws Exception {
+for (int i = 0; i < 5; i++) {
+  testPrintDispatcherEventDetailsAvoidDeadLoopInternal();
+}
+  }
+
+  public void testPrintDispatcherEventDetail

[hadoop] branch branch-3.3 updated: YARN-10642. Race condition: AsyncDispatcher can get stuck by the changes introduced in YARN-8995. Contributed by zhengchenyu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new eb05d49  YARN-10642. Race condition: AsyncDispatcher can get stuck by 
the changes introduced in YARN-8995. Contributed by zhengchenyu.
eb05d49 is described below

commit eb05d49349d223052db74f528b239a5392584934
Author: Peter Bacsko 
AuthorDate: Mon Mar 8 12:36:08 2021 +0100

YARN-10642. Race condition: AsyncDispatcher can get stuck by the changes 
introduced in YARN-8995. Contributed by zhengchenyu.
---
 .../apache/hadoop/yarn/event/AsyncDispatcher.java  | 19 ---
 .../hadoop/yarn/event/TestAsyncDispatcher.java | 58 +-
 2 files changed, 68 insertions(+), 9 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 79ad464..f9deab0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -20,11 +20,11 @@ package org.apache.hadoop.yarn.event;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -268,11 +268,16 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   }
 
   class GenericEventHandler implements EventHandler {
-private void printEventQueueDetails(BlockingQueue queue) {
-  Map counterMap = eventQueue.stream().
-  collect(Collectors.
-  groupingBy(e -> e.getType(), Collectors.counting())
-  );
+private void printEventQueueDetails() {
+  Iterator iterator = eventQueue.iterator();
+  Map counterMap = new HashMap<>();
+  while (iterator.hasNext()) {
+Enum eventType = iterator.next().getType();
+if (!counterMap.containsKey(eventType)) {
+  counterMap.put(eventType, 0L);
+}
+counterMap.put(eventType, counterMap.get(eventType) + 1);
+  }
   for (Map.Entry entry : counterMap.entrySet()) {
 long num = entry.getValue();
 LOG.info("Event type: " + entry.getKey()
@@ -295,7 +300,7 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   if (qSize != 0 && qSize % detailsInterval == 0
   && lastEventDetailsQueueSizeLogged != qSize) {
 lastEventDetailsQueueSizeLogged = qSize;
-printEventQueueDetails(eventQueue);
+printEventQueueDetails();
 printTrigger = true;
   }
   int remCapacity = eventQueue.remainingCapacity();
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
index 762e228..55ddd12 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
@@ -97,12 +97,23 @@ public class TestAsyncDispatcher {
   }
 
   private static class TestHandler implements EventHandler {
+
+private long sleepTime = 1500;
+
+TestHandler() {
+}
+
+TestHandler(long sleepTime) {
+  this.sleepTime = sleepTime;
+}
+
 @Override
 public void handle(Event event) {
   try {
 // As long as 1 events queued
-Thread.sleep(1500);
-  } catch (InterruptedException e) {}
+Thread.sleep(this.sleepTime);
+  } catch (InterruptedException e) {
+  }
 }
   }
 
@@ -170,11 +181,54 @@ public class TestAsyncDispatcher {
   //Make sure more than one event to take
   verify(log, atLeastOnce()).
   info("Latest dispatch event type: TestEventType");
+} finally {
+  //... restore logger object
+  logger.set(null, oldLog);
   dispatcher.stop();
+}
+  }
+
+  //Test print dispatcher details when the blocking queue is heavy
+  @Test(timeout = 6)
+  public void testPrintDispatcherEventDetailsAvoidDeadLoop() throws Exception {
+for (int i = 0; i < 5; i++) {
+  testPrintDispatcherEventDetailsAvoidDeadLoopInternal();
+}
+  }
+
+  public void testPrintDispatcherEventDetailsAvoidDeadLoopInternal()
+

[hadoop] branch trunk updated: YARN-10658. CapacityScheduler QueueInfo add queue path field to avoid ambiguous QueueName. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 04cd311  YARN-10658. CapacityScheduler QueueInfo add queue path field 
to avoid ambiguous QueueName. Contributed by Qi Zhu.
04cd311 is described below

commit 04cd3115ba3f078eea7e76c3bb7d5228266378d8
Author: Peter Bacsko 
AuthorDate: Mon Mar 8 12:29:44 2021 +0100

YARN-10658. CapacityScheduler QueueInfo add queue path field to avoid 
ambiguous QueueName. Contributed by Qi Zhu.
---
 .../apache/hadoop/yarn/api/records/QueueInfo.java  | 26 
 .../src/main/proto/yarn_protos.proto   |  1 +
 .../apache/hadoop/yarn/client/cli/QueueCLI.java|  2 ++
 .../hadoop/yarn/client/ProtocolHATestBase.java |  2 +-
 .../apache/hadoop/yarn/client/cli/TestYarnCLI.java | 10 +--
 .../yarn/api/records/impl/pb/QueueInfoPBImpl.java  | 16 ++
 .../apache/hadoop/yarn/api/TestPBImplRecords.java  |  3 +-
 .../scheduler/capacity/AbstractCSQueue.java|  1 +
 .../TestConfigurationMutationACLPolicies.java  | 17 ++-
 .../scheduler/TestSchedulerApplicationAttempt.java |  3 +-
 .../scheduler/capacity/TestCapacityScheduler.java  |  6 
 .../TestCapacitySchedulerNewQueueAutoCreation.java | 35 ++
 .../scheduler/capacity/TestLeafQueue.java  | 15 ++
 13 files changed, 114 insertions(+), 23 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
index 803adad..eb703cd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
@@ -54,7 +54,8 @@ public abstract class QueueInfo {
   
   @Private
   @Unstable
-  public static QueueInfo newInstance(String queueName, float capacity,
+  public static QueueInfo newInstance(String queueName,
+  String queuePath, float capacity,
   float maximumCapacity, float currentCapacity,
   List childQueues, List applications,
   QueueState queueState, Set accessibleNodeLabels,
@@ -62,6 +63,7 @@ public abstract class QueueInfo {
   boolean preemptionDisabled, float weight) {
 QueueInfo queueInfo = Records.newRecord(QueueInfo.class);
 queueInfo.setQueueName(queueName);
+queueInfo.setQueuePath(queuePath);
 queueInfo.setCapacity(capacity);
 queueInfo.setMaximumCapacity(maximumCapacity);
 queueInfo.setCurrentCapacity(currentCapacity);
@@ -78,14 +80,15 @@ public abstract class QueueInfo {
 
   @Private
   @Unstable
-  public static QueueInfo newInstance(String queueName, float capacity,
+  public static QueueInfo newInstance(String queueName,
+  String queuePath, float capacity,
   float maximumCapacity, float currentCapacity,
   List childQueues, List applications,
   QueueState queueState, Set accessibleNodeLabels,
   String defaultNodeLabelExpression, QueueStatistics queueStatistics,
   boolean preemptionDisabled, float weight,
   Map queueConfigurations) {
-QueueInfo queueInfo = QueueInfo.newInstance(queueName, capacity,
+QueueInfo queueInfo = QueueInfo.newInstance(queueName, queuePath, capacity,
 maximumCapacity, currentCapacity,
 childQueues, applications,
 queueState, accessibleNodeLabels,
@@ -97,7 +100,8 @@ public abstract class QueueInfo {
 
   @Private
   @Unstable
-  public static QueueInfo newInstance(String queueName, float capacity,
+  public static QueueInfo newInstance(String queueName,
+  String queuePath, float capacity,
   float maximumCapacity, float currentCapacity,
   List childQueues, List applications,
   QueueState queueState, Set accessibleNodeLabels,
@@ -105,7 +109,7 @@ public abstract class QueueInfo {
   boolean preemptionDisabled, float weight,
   Map queueConfigurations,
   boolean intraQueuePreemptionDisabled) {
-QueueInfo queueInfo = QueueInfo.newInstance(queueName, capacity,
+QueueInfo queueInfo = QueueInfo.newInstance(queueName, queuePath, capacity,
 maximumCapacity, currentCapacity,
 childQueues, applications,
 queueState, accessibleNodeLabels,
@@ -126,6 +130,18 @@ public abstract class QueueInfo {
   @Private
   @Unstable
   public abstract void setQueueName(String queueName);
+
+  /**
+   * Get the path of the queue.
+   * @return path of the queue
+   */
+  @Public
+  @Stable
+  public abstract String getQueuePath();
+
+  @Private
+  @Unstable
+  public abstract void setQueuePath(String queuePath);
   
   /**
* Get the configured capacity of the queue.
diff --git 
a/hadoop-yarn-project/hadoop

[hadoop] branch branch-3.3 updated: YARN-10672. All testcases in TestReservations are flaky. Contributed by Szilard Nemeth.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 066f89a  YARN-10672. All testcases in TestReservations are flaky. 
Contributed by Szilard Nemeth.
066f89a is described below

commit 066f89af019811bc3b16703f89125a10d2b2a07e
Author: Peter Bacsko 
AuthorDate: Mon Mar 8 11:42:59 2021 +0100

YARN-10672. All testcases in TestReservations are flaky. Contributed by 
Szilard Nemeth.
---
 .../server/resourcemanager/scheduler/capacity/TestReservations.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index a9cee2d..1168f64 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -105,7 +105,6 @@ public class TestReservations {
 CapacityScheduler spyCs = new CapacityScheduler();
 cs = spy(spyCs);
 rmContext = TestUtils.getMockRMContext();
-
   }
 
   private void setup(CapacitySchedulerConfiguration csConf) throws Exception {
@@ -114,6 +113,9 @@ public class TestReservations {
 
   private void setup(CapacitySchedulerConfiguration csConf,
   boolean addUserLimits) throws Exception {
+//All stub calls on the spy object of the 'cs' field should happen
+//before cs.start() is invoked. See YARN-10672 for more details.
+when(cs.getNumClusterNodes()).thenReturn(3);
 
 csConf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, 
true);
 final String newRoot = "root" + System.currentTimeMillis();
@@ -156,8 +158,6 @@ public class TestReservations {
 cs.setRMContext(spyRMContext);
 cs.init(csConf);
 cs.start();
-
-when(cs.getNumClusterNodes()).thenReturn(3);
   }
 
   private static final String A = "a";


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10672. All testcases in TestReservations are flaky. Contributed By Szilard Nemeth.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new c6b30a5  YARN-10672. All testcases in TestReservations are flaky. 
Contributed By Szilard Nemeth.
c6b30a5 is described below

commit c6b30a59dd20db39f34db8ce61ec152d7a90dad6
Author: Peter Bacsko 
AuthorDate: Fri Mar 5 23:13:35 2021 +0100

YARN-10672. All testcases in TestReservations are flaky. Contributed By 
Szilard Nemeth.
---
 .../server/resourcemanager/scheduler/capacity/TestReservations.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index a9cee2d..1168f64 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -105,7 +105,6 @@ public class TestReservations {
 CapacityScheduler spyCs = new CapacityScheduler();
 cs = spy(spyCs);
 rmContext = TestUtils.getMockRMContext();
-
   }
 
   private void setup(CapacitySchedulerConfiguration csConf) throws Exception {
@@ -114,6 +113,9 @@ public class TestReservations {
 
   private void setup(CapacitySchedulerConfiguration csConf,
   boolean addUserLimits) throws Exception {
+//All stub calls on the spy object of the 'cs' field should happen
+//before cs.start() is invoked. See YARN-10672 for more details.
+when(cs.getNumClusterNodes()).thenReturn(3);
 
 csConf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, 
true);
 final String newRoot = "root" + System.currentTimeMillis();
@@ -156,8 +158,6 @@ public class TestReservations {
 cs.setRMContext(spyRMContext);
 cs.init(csConf);
 cs.start();
-
-when(cs.getNumClusterNodes()).thenReturn(3);
   }
 
   private static final String A = "a";


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10640. Adjust the queue Configured capacity to Configured weight number for weight mode in UI. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new e82e7c5  YARN-10640. Adjust the queue Configured capacity to 
Configured weight number for weight mode in UI. Contributed by Qi Zhu.
e82e7c5 is described below

commit e82e7c597a130bb7a4e4b3132be85b7099961a2d
Author: Peter Bacsko 
AuthorDate: Fri Mar 5 15:56:56 2021 +0100

YARN-10640. Adjust the queue Configured capacity to Configured weight 
number for weight mode in UI. Contributed by Qi Zhu.
---
 .../yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java | 8 ++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 47c888d..8212a72 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -159,8 +159,12 @@ class CapacitySchedulerPage extends RmView {
   __("Used Capacity:",
   appendPercent(resourceUsages.getUsed(),
   capacities.getUsedCapacity() / 100))
-  .__("Configured Capacity:",
-  capacities.getConfiguredMinResource() == null ?
+  .__(capacities.getWeight() != -1 ?
+  "Configured Weight:" :
+  "Configured Capacity:",
+  capacities.getWeight() != -1 ?
+  capacities.getWeight() :
+  capacities.getConfiguredMinResource() == null ?
   Resources.none().toString() :
   capacities.getConfiguredMinResource().toString())
   .__("Configured Max Capacity:",


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10642. Race condition: AsyncDispatcher can get stuck by the changes introduced in YARN-8995. Contributed by zhengchenyu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new e7cad38  YARN-10642. Race condition: AsyncDispatcher can get stuck by 
the changes introduced in YARN-8995. Contributed by zhengchenyu.
e7cad38 is described below

commit e7cad3811fd14ff0b963423a24a9012ce5fb07a9
Author: Peter Bacsko 
AuthorDate: Fri Mar 5 13:50:45 2021 +0100

YARN-10642. Race condition: AsyncDispatcher can get stuck by the changes 
introduced in YARN-8995. Contributed by zhengchenyu.
---
 .../apache/hadoop/yarn/event/AsyncDispatcher.java  | 19 ---
 .../hadoop/yarn/event/TestAsyncDispatcher.java | 58 +-
 2 files changed, 68 insertions(+), 9 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 79ad464..f9deab0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -20,11 +20,11 @@ package org.apache.hadoop.yarn.event;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -268,11 +268,16 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   }
 
   class GenericEventHandler implements EventHandler {
-private void printEventQueueDetails(BlockingQueue queue) {
-  Map counterMap = eventQueue.stream().
-  collect(Collectors.
-  groupingBy(e -> e.getType(), Collectors.counting())
-  );
+private void printEventQueueDetails() {
+  Iterator iterator = eventQueue.iterator();
+  Map counterMap = new HashMap<>();
+  while (iterator.hasNext()) {
+Enum eventType = iterator.next().getType();
+if (!counterMap.containsKey(eventType)) {
+  counterMap.put(eventType, 0L);
+}
+counterMap.put(eventType, counterMap.get(eventType) + 1);
+  }
   for (Map.Entry entry : counterMap.entrySet()) {
 long num = entry.getValue();
 LOG.info("Event type: " + entry.getKey()
@@ -295,7 +300,7 @@ public class AsyncDispatcher extends AbstractService 
implements Dispatcher {
   if (qSize != 0 && qSize % detailsInterval == 0
   && lastEventDetailsQueueSizeLogged != qSize) {
 lastEventDetailsQueueSizeLogged = qSize;
-printEventQueueDetails(eventQueue);
+printEventQueueDetails();
 printTrigger = true;
   }
   int remCapacity = eventQueue.remainingCapacity();
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
index 762e228..55ddd12 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
@@ -97,12 +97,23 @@ public class TestAsyncDispatcher {
   }
 
   private static class TestHandler implements EventHandler {
+
+private long sleepTime = 1500;
+
+TestHandler() {
+}
+
+TestHandler(long sleepTime) {
+  this.sleepTime = sleepTime;
+}
+
 @Override
 public void handle(Event event) {
   try {
 // As long as 1 events queued
-Thread.sleep(1500);
-  } catch (InterruptedException e) {}
+Thread.sleep(this.sleepTime);
+  } catch (InterruptedException e) {
+  }
 }
   }
 
@@ -170,11 +181,54 @@ public class TestAsyncDispatcher {
   //Make sure more than one event to take
   verify(log, atLeastOnce()).
   info("Latest dispatch event type: TestEventType");
+} finally {
+  //... restore logger object
+  logger.set(null, oldLog);
   dispatcher.stop();
+}
+  }
+
+  //Test print dispatcher details when the blocking queue is heavy
+  @Test(timeout = 6)
+  public void testPrintDispatcherEventDetailsAvoidDeadLoop() throws Exception {
+for (int i = 0; i < 5; i++) {
+  testPrintDispatcherEventDetailsAvoidDeadLoopInternal();
+}
+  }
+
+  public void testPrintDispatcherEventDetailsAvoidDeadLoopInternal()
+  thro

[hadoop] branch trunk updated: YARN-10639. Queueinfo related capacity, should adjusted to weight mode. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new e19c009  YARN-10639. Queueinfo related capacity, should adjusted to 
weight mode. Contributed by Qi Zhu.
e19c009 is described below

commit e19c00925f38346dc1291cdced108ecc0206fb74
Author: Peter Bacsko 
AuthorDate: Fri Mar 5 13:18:06 2021 +0100

YARN-10639. Queueinfo related capacity, should adjusted to weight mode. 
Contributed by Qi Zhu.
---
 .../apache/hadoop/yarn/api/records/QueueInfo.java  | 23 ---
 .../src/main/proto/yarn_protos.proto   |  1 +
 .../apache/hadoop/yarn/client/cli/QueueCLI.java|  2 +
 .../hadoop/yarn/client/ProtocolHATestBase.java |  5 ++-
 .../apache/hadoop/yarn/client/cli/TestYarnCLI.java |  8 +++-
 .../yarn/api/records/impl/pb/QueueInfoPBImpl.java  | 12 ++
 .../apache/hadoop/yarn/api/TestPBImplRecords.java  |  2 +-
 .../scheduler/capacity/AbstractCSQueue.java|  1 +
 .../TestConfigurationMutationACLPolicies.java  |  2 +-
 .../scheduler/TestSchedulerApplicationAttempt.java |  2 +-
 .../capacity/TestCapacitySchedulerWeightMode.java  | 45 +++---
 .../scheduler/capacity/TestLeafQueue.java  |  2 +-
 12 files changed, 87 insertions(+), 18 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
index 57ea9bf..803adad 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
@@ -59,7 +59,7 @@ public abstract class QueueInfo {
   List childQueues, List applications,
   QueueState queueState, Set accessibleNodeLabels,
   String defaultNodeLabelExpression, QueueStatistics queueStatistics,
-  boolean preemptionDisabled) {
+  boolean preemptionDisabled, float weight) {
 QueueInfo queueInfo = Records.newRecord(QueueInfo.class);
 queueInfo.setQueueName(queueName);
 queueInfo.setCapacity(capacity);
@@ -72,6 +72,7 @@ public abstract class QueueInfo {
 queueInfo.setDefaultNodeLabelExpression(defaultNodeLabelExpression);
 queueInfo.setQueueStatistics(queueStatistics);
 queueInfo.setPreemptionDisabled(preemptionDisabled);
+queueInfo.setWeight(weight);
 return queueInfo;
   }
 
@@ -82,14 +83,14 @@ public abstract class QueueInfo {
   List childQueues, List applications,
   QueueState queueState, Set accessibleNodeLabels,
   String defaultNodeLabelExpression, QueueStatistics queueStatistics,
-  boolean preemptionDisabled,
+  boolean preemptionDisabled, float weight,
   Map queueConfigurations) {
 QueueInfo queueInfo = QueueInfo.newInstance(queueName, capacity,
 maximumCapacity, currentCapacity,
 childQueues, applications,
 queueState, accessibleNodeLabels,
 defaultNodeLabelExpression, queueStatistics,
-preemptionDisabled);
+preemptionDisabled, weight);
 queueInfo.setQueueConfigurations(queueConfigurations);
 return queueInfo;
   }
@@ -101,7 +102,7 @@ public abstract class QueueInfo {
   List childQueues, List applications,
   QueueState queueState, Set accessibleNodeLabels,
   String defaultNodeLabelExpression, QueueStatistics queueStatistics,
-  boolean preemptionDisabled,
+  boolean preemptionDisabled, float weight,
   Map queueConfigurations,
   boolean intraQueuePreemptionDisabled) {
 QueueInfo queueInfo = QueueInfo.newInstance(queueName, capacity,
@@ -109,7 +110,7 @@ public abstract class QueueInfo {
 childQueues, applications,
 queueState, accessibleNodeLabels,
 defaultNodeLabelExpression, queueStatistics,
-preemptionDisabled, queueConfigurations);
+preemptionDisabled, weight, queueConfigurations);
 queueInfo.setIntraQueuePreemptionDisabled(intraQueuePreemptionDisabled);
 return queueInfo;
   }
@@ -137,6 +138,18 @@ public abstract class QueueInfo {
   @Private
   @Unstable
   public abstract void setCapacity(float capacity);
+
+  /**
+   * Get the configured weight of the queue.
+   * @return configured weight of the queue
+   */
+  @Public
+  @Stable
+  public abstract float getWeight();
+
+  @Private
+  @Unstable
+  public abstract void setWeight(float weight);
   
   /**
* Get the maximum capacity of the queue.
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index d7ca2a5..467f26a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto

[hadoop] branch trunk updated: YARN-10532. Capacity Scheduler Auto Queue Creation: Allow auto delete queue when queue is not being used. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 6699198  YARN-10532. Capacity Scheduler Auto Queue Creation: Allow 
auto delete queue when queue is not being used. Contributed by Qi Zhu.
6699198 is described below

commit 6699198b54bf6360c164a6ce7552c8b91a318c59
Author: Peter Bacsko 
AuthorDate: Thu Mar 4 17:18:35 2021 +0100

YARN-10532. Capacity Scheduler Auto Queue Creation: Allow auto delete queue 
when queue is not being used. Contributed by Qi Zhu.
---
 .../scheduler/capacity/AbstractCSQueue.java|  47 
 .../capacity/AutoCreatedQueueDeletionPolicy.java   | 163 +++
 .../scheduler/capacity/CapacityScheduler.java  |  64 -
 .../capacity/CapacitySchedulerConfiguration.java   |  56 
 .../scheduler/capacity/LeafQueue.java  |  10 +
 .../scheduler/capacity/ParentQueue.java|  34 ++-
 ...ype.java => AutoCreatedQueueDeletionEvent.java} |  49 +---
 .../scheduler/event/SchedulerEventType.java|   5 +-
 .../monitor/TestSchedulingMonitor.java |  43 +++
 .../TestAutoCreatedQueueDeletionPolicy.java| 184 +
 .../TestCapacitySchedulerNewQueueAutoCreation.java | 303 -
 11 files changed, 910 insertions(+), 48 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 28e2d54..e5380fa 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -29,6 +29,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import 
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.util.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -154,6 +155,10 @@ public abstract class AbstractCSQueue implements CSQueue {
   // is it a dynamic queue?
   private boolean dynamicQueue = false;
 
+  // The timestamp of the last submitted application to this queue.
+  // Only applies to dynamic queues.
+  private long lastSubmittedTimestamp;
+
   public AbstractCSQueue(CapacitySchedulerContext cs,
   String queueName, CSQueue parent, CSQueue old) throws IOException {
 this(cs, cs.getConfiguration(), queueName, parent, old);
@@ -1642,4 +1647,46 @@ public abstract class AbstractCSQueue implements CSQueue 
{
   return "capacity=" + queueCapacities.getCapacity();
 }
   }
+
+  public boolean isEligibleForAutoDeletion() {
+return false;
+  }
+
+  public boolean isInactiveDynamicQueue() {
+long idleDurationSeconds =
+(Time.monotonicNow() - getLastSubmittedTimestamp())/1000;
+return isDynamicQueue() && isEligibleForAutoDeletion() &&
+(idleDurationSeconds > this.csContext.getConfiguration().
+getAutoExpiredDeletionTime());
+  }
+
+  public void updateLastSubmittedTimeStamp() {
+writeLock.lock();
+try {
+  this.lastSubmittedTimestamp = Time.monotonicNow();
+} finally {
+  writeLock.unlock();
+}
+  }
+
+  public long getLastSubmittedTimestamp() {
+readLock.lock();
+
+try {
+  return lastSubmittedTimestamp;
+} finally {
+  readLock.unlock();
+}
+  }
+
+  @VisibleForTesting
+  public void setLastSubmittedTimestamp(long lastSubmittedTimestamp) {
+writeLock.lock();
+try {
+  this.lastSubmittedTimestamp = lastSubmittedTimestamp;
+} finally {
+  writeLock.unlock();
+}
+  }
+
 }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueDeletionPolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueDeletionPolicy.java
new file mode 100644
index 000..4b47bb4
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueDe

[hadoop] branch trunk updated: YARN-10623. Capacity scheduler should support refresh queue automatically by a thread policy. Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new a85aeee  YARN-10623. Capacity scheduler should support refresh queue 
automatically by a thread policy. Contributed by Qi Zhu.
a85aeee is described below

commit a85aeee876f850c192c7039b763b34ef4a2dc0cb
Author: Peter Bacsko 
AuthorDate: Thu Mar 4 12:23:11 2021 +0100

YARN-10623. Capacity scheduler should support refresh queue automatically 
by a thread policy. Contributed by Qi Zhu.
---
 .../yarn/FileSystemBasedConfigurationProvider.java |   8 +
 .../scheduler/capacity/CapacityScheduler.java  |   4 +
 .../capacity/CapacitySchedulerConfiguration.java   |  12 +
 .../QueueConfigurationAutoRefreshPolicy.java   | 196 +
 .../TestQueueConfigurationAutoRefreshPolicy.java   | 308 +
 5 files changed, 528 insertions(+)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
index 3532d13..156468e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
@@ -92,4 +92,12 @@ public class FileSystemBasedConfigurationProvider
   public synchronized void closeInternal() throws Exception {
 fs.close();
   }
+
+  public FileSystem getFs() {
+return fs;
+  }
+
+  public Path getConfigDir() {
+return configDir;
+  }
 }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 36f831d..ee91b0c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -3396,6 +3396,10 @@ public class CapacityScheduler extends
 return null;
   }
 
+  public CSConfigurationProvider getCsConfProvider() {
+return csConfProvider;
+  }
+
   @Override
   public void resetSchedulerMetrics() {
 CapacitySchedulerMetrics.destroy();
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 8e60596..b66ab85 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -2201,6 +2201,18 @@ public class CapacitySchedulerConfiguration extends 
ReservationSchedulerConfigur
   1500L;
 
   /**
+   * Time in milliseconds between invocations
+   * of QueueConfigurationAutoRefreshPolicy.
+   */
+  @Private
+  public static final String QUEUE_AUTO_REFRESH_MONITORING_INTERVAL =
+  PREFIX + "queue.auto.refresh.monitoring-interval";
+
+  @Private
+  public static final long DEFAULT_QUEUE_AUTO_REFRESH_MONITORING_INTERVAL =
+  5000L;
+
+  /**
* Queue Management computation policy for Auto Created queues
* @param queue The queue's path
* @return Configured policy class name
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueConfigurationAutoRefreshPolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueConfigurationAutoRefreshPolicy.java
new file mode 100644
index 000..0ae0777
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-y

[hadoop] branch trunk updated: YARN-10655. Limit queue creation depth relative to its first static parent. Contributed by Andras Gyori.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 7e8040e  YARN-10655. Limit queue creation depth relative to its first 
static parent. Contributed by Andras Gyori.
7e8040e is described below

commit 7e8040e6adccad55a603c1ecd513a02e7edc7425
Author: Peter Bacsko 
AuthorDate: Wed Mar 3 17:44:30 2021 +0100

YARN-10655. Limit queue creation depth relative to its first static parent. 
Contributed by Andras Gyori.
---
 .../CapacitySchedulerAutoQueueHandler.java | 40 +++---
 .../TestCapacitySchedulerNewQueueAutoCreation.java | 20 +++
 2 files changed, 47 insertions(+), 13 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java
index e847737..898b075 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java
@@ -55,34 +55,43 @@ public class CapacitySchedulerAutoQueueHandler {
 List parentsToCreate = new ArrayList<>();
 
 ApplicationPlacementContext queueCandidateContext = parentContext;
-CSQueue existingQueueCandidate = getQueue(
+CSQueue firstExistingQueue = getQueue(
 queueCandidateContext.getFullQueuePath());
 
-while (existingQueueCandidate == null) {
+while (firstExistingQueue == null) {
   parentsToCreate.add(queueCandidateContext);
   queueCandidateContext = CSQueueUtils.extractQueuePath(
   queueCandidateContext.getParentQueue());
-  existingQueueCandidate = getQueue(
+  firstExistingQueue = getQueue(
   queueCandidateContext.getFullQueuePath());
 }
 
+CSQueue firstExistingStaticQueue = firstExistingQueue;
+// Include the LeafQueue in the distance
+int firstStaticParentDistance = parentsToCreate.size() + 1;
+
+while(isNonStaticParent(firstExistingStaticQueue)) {
+  queueCandidateContext = CSQueueUtils.extractQueuePath(
+  queueCandidateContext.getParentQueue());
+  firstExistingStaticQueue = getQueue(
+  queueCandidateContext.getFullQueuePath());
+  ++firstStaticParentDistance;
+}
+
 // Reverse the collection to to represent the hierarchy to be created
 // from highest to lowest level
 Collections.reverse(parentsToCreate);
 
-if (!(existingQueueCandidate instanceof ParentQueue)) {
+if (!(firstExistingQueue instanceof ParentQueue)) {
   throw new SchedulerDynamicEditException(
   "Could not auto create hierarchy of "
   + queue.getFullQueuePath() + ". Queue "
-  + existingQueueCandidate.getQueuePath() +
+  + firstExistingQueue.getQueuePath() +
   " is not a ParentQueue."
   );
 }
-ParentQueue existingParentQueue = (ParentQueue) existingQueueCandidate;
+ParentQueue existingParentQueue = (ParentQueue) firstExistingQueue;
 int depthLimit = extractDepthLimit(existingParentQueue);
-// The number of levels to be created including the LeafQueue
-// (which is last)
-int levelsToCreate = parentsToCreate.size() + 1;
 
 if (depthLimit == 0) {
   throw new SchedulerDynamicEditException("Auto creation of queue " +
@@ -90,12 +99,12 @@ public class CapacitySchedulerAutoQueueHandler {
   + existingParentQueue.getQueuePath());
 }
 
-if (levelsToCreate > depthLimit) {
+if (firstStaticParentDistance > depthLimit) {
   throw new SchedulerDynamicEditException(
   "Could not auto create queue " + queue.getFullQueuePath()
-  + ". In order to create the desired queue hierarchy, " +
-  levelsToCreate + " levels of queues would need " +
-  "to be created, which is above the limit.");
+  + ". The distance of the LeafQueue from the first static " +
+  "ParentQueue is" + firstStaticParentDistance + ", which is " +
+  "above the limit.");
 }
 
 for (ApplicationPlacementContext current : parentsToCreate) {
@@ -123,4 +132,9 @@ public class CapacitySchedulerAutoQueueHandler {
   private CSQueue g

[hadoop] branch trunk updated: YARN-10627. Extend logging to give more information about weight mode. Contributed by Benjamin Teke.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 947b504  YARN-10627. Extend logging to give more information about 
weight mode. Contributed by Benjamin Teke.
947b504 is described below

commit 947b50489d5c8b03969bbdb68e65e7f84f074101
Author: Peter Bacsko 
AuthorDate: Fri Feb 26 21:57:54 2021 +0100

YARN-10627. Extend logging to give more information about weight mode. 
Contributed by Benjamin Teke.
---
 .../scheduler/capacity/AbstractCSQueue.java|  9 +++
 .../scheduler/capacity/LeafQueue.java  | 19 +++--
 .../scheduler/capacity/ParentQueue.java|  6 +-
 .../capacity/TestCapacitySchedulerWeightMode.java  | 84 +-
 4 files changed, 95 insertions(+), 23 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 12ce05f..28e2d54 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -1633,4 +1633,13 @@ public abstract class AbstractCSQueue implements CSQueue 
{
   writeLock.unlock();
 }
   }
+
+  protected String getCapacityOrWeightString() {
+if (queueCapacities.getWeight() != -1) {
+  return "weight=" + queueCapacities.getWeight() + ", " +
+  "normalizedWeight=" + queueCapacities.getNormalizedWeight();
+} else {
+  return "capacity=" + queueCapacities.getCapacity();
+}
+  }
 }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 6bf8d0a..104a89c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -268,9 +268,8 @@ public class LeafQueue extends AbstractCSQueue {
   usersManager.updateUserWeights();
 
   LOG.info(
-  "Initializing " + getQueuePath() + "\n" + "capacity = "
-  + queueCapacities.getCapacity()
-  + " [= (float) configuredCapacity / 100 ]" + "\n"
+  "Initializing " + getQueuePath() + "\n" +
+  getExtendedCapacityOrWeightString() + "\n"
   + "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity()
   + " [= parentAbsoluteCapacity * capacity ]" + "\n"
   + "maxCapacity = " + queueCapacities.getMaximumCapacity()
@@ -486,7 +485,7 @@ public class LeafQueue extends AbstractCSQueue {
   public String toString() {
 readLock.lock();
 try {
-  return getQueuePath() + ": " + "capacity=" + 
queueCapacities.getCapacity()
+  return getQueuePath() + ": " + getCapacityOrWeightString()
   + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity()
   + ", " + "usedResources=" + queueUsage.getUsed() + ", "
   + "usedCapacity=" + getUsedCapacity() + ", " + 
"absoluteUsedCapacity="
@@ -499,7 +498,19 @@ public class LeafQueue extends AbstractCSQueue {
 } finally {
   readLock.unlock();
 }
+  }
 
+  protected String getExtendedCapacityOrWeightString() {
+if (queueCapacities.getWeight() != -1) {
+  return "weight = " + queueCapacities.getWeight()
+  + " [= (float) configuredCapacity (with w suffix)] " + "\n"
+  + "normalizedWeight = " + queueCapacities.getNormalizedWeight()
+  + " [= (float) configuredCapacity / sum(configuredCapacity of " +
+  "all queues under the parent)]";
+   

[hadoop] branch trunk updated: YARN-10609. Update the document for YARN-10531(Be able to disable user limit factor for CapacityScheduler Leaf Queue). Contributed by Qi Zhu.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new a89c624  YARN-10609. Update the document for YARN-10531(Be able to 
disable user limit factor for CapacityScheduler Leaf Queue). Contributed by Qi 
Zhu.
a89c624 is described below

commit a89c624002fc4fc956f70bd6b641511bee6c2e35
Author: Peter Bacsko 
AuthorDate: Wed Feb 24 20:41:35 2021 +0100

YARN-10609. Update the document for YARN-10531(Be able to disable user 
limit factor for CapacityScheduler Leaf Queue). Contributed by Qi Zhu.
---
 .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index e5b23a7..8acf723 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -126,7 +126,7 @@ Configuration
 | `yarn.scheduler.capacity..capacity` | Queue *capacity* in 
percentage (%) as a float (e.g. 12.5) OR as absolute resource queue minimum 
capacity. The sum of capacities for all queues, at each level, must be equal to 
100. However if absolute resource is configured, sum of absolute resources of 
child queues could be less than it's parent absolute resource capacity. 
Applications in the queue may consume more resources than the queue's capacity 
if there are free resources, provid [...]
 | `yarn.scheduler.capacity..maximum-capacity` | Maximum queue 
capacity in percentage (%) as a float OR as absolute resource queue maximum 
capacity. This limits the *elasticity* for applications in the queue. 1) Value 
is between 0 and 100. 2) Admin needs to make sure absolute maximum capacity >= 
absolute capacity for each queue. Also, setting this value to -1 sets maximum 
capacity to 100%. |
 | `yarn.scheduler.capacity..minimum-user-limit-percent` | Each 
queue enforces a limit on the percentage of resources allocated to a user at 
any given time, if there is demand for resources. The user limit can vary 
between a minimum and maximum value. The former (the minimum value) is set to 
this property value and the latter (the maximum value) depends on the number of 
users who have submitted applications. For e.g., suppose the value of this 
property is 25. If two users have [...]
-| `yarn.scheduler.capacity..user-limit-factor` | The multiple of 
the queue capacity which can be configured to allow a single user to acquire 
more resources. By default this is set to 1 which ensures that a single user 
can never take more than the queue's configured capacity irrespective of how 
idle the cluster is. Value is specified as a float. |
+| `yarn.scheduler.capacity..user-limit-factor` | User limit factor 
provides a way to control the max amount of resources that a single user can 
consume. It is the multiple of the queue's capacity. By default this is set to 
1 which ensures that a single user can never take more than the queue's 
configured capacity irrespective of how idle the cluster is. Increasing it 
means a single user can use more than the minimum capacity of the cluster, 
while decreasing it results in lowe [...]
 | `yarn.scheduler.capacity..maximum-allocation-mb` | The per queue 
maximum limit of memory to allocate to each container request at the Resource 
Manager. This setting overrides the cluster configuration 
`yarn.scheduler.maximum-allocation-mb`. This value must be smaller than or 
equal to the cluster maximum. |
 | `yarn.scheduler.capacity..maximum-allocation-vcores` | The per 
queue maximum limit of virtual cores to allocate to each container request at 
the Resource Manager. This setting overrides the cluster configuration 
`yarn.scheduler.maximum-allocation-vcores`. This value must be smaller than or 
equal to the cluster maximum. |
 | `yarn.scheduler.capacity..user-settings..weight` | 
This floating point value is used when calculating the user limit resource 
values for users in a queue. This value will weight each user more or less than 
the other users in the queue. For example, if user A should receive 50% more 
resources in a queue than users B and C, this property will be set to 1.5 for 
user A.  Users B and C will default to 1.0. |


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10513. CS Flexible Auto Queue Creation RM UIv2 modifications. Contributed by Andras Gyori.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new b722c52  YARN-10513. CS Flexible Auto Queue Creation RM UIv2 
modifications. Contributed by Andras Gyori.
b722c52 is described below

commit b722c52f4ede4c19c76a7947583fd6104e2c908a
Author: Peter Bacsko 
AuthorDate: Mon Feb 22 12:55:45 2021 +0100

YARN-10513. CS Flexible Auto Queue Creation RM UIv2 modifications. 
Contributed by Andras Gyori.
---
 .../yarn-queue-partition-capacity-labels.js| 14 -
 .../webapp/app/models/yarn-queue/capacity-queue.js | 13 
 .../app/serializers/yarn-queue/capacity-queue.js   |  5 +++
 .../yarn-queue-partition-capacity-labels.hbs   | 22 -
 .../src/main/webapp/app/templates/yarn-queue.hbs   | 36 +-
 5 files changed, 80 insertions(+), 10 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/yarn-queue-partition-capacity-labels.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/yarn-queue-partition-capacity-labels.js
index e7f9c03..788155c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/yarn-queue-partition-capacity-labels.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/yarn-queue-partition-capacity-labels.js
@@ -37,12 +37,24 @@ export default Ember.Component.extend({
 const userLimit = queue.get("userLimit");
 const userLimitFactor = queue.get("userLimitFactor");
 const isLeafQueue = queue.get("isLeafQueue");
+const isWeightMode = queue.get("isWeightMode");
+const isFlexibleDynamicQueue = queue.get("isFlexibleDynamicQueue");
+const weight = queue.get("weight");
+const orderingPolicy = queue.get("orderingPolicy");
+const normalizedWeight = queue.get("normalizedWeight");
+const creationMethod = queue.get("creationMethod");
 
 return {
   ...partitionMap[filteredParition],
   userLimit,
   userLimitFactor,
-  isLeafQueue
+  isLeafQueue,
+  isWeightMode,
+  weight,
+  normalizedWeight,
+  orderingPolicy,
+  creationMethod,
+  isFlexibleDynamicQueue
 };
   }
 });
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
index ebcccb9..885c6a5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
@@ -21,6 +21,7 @@ import Converter from 'yarn-ui/utils/converter';
 
 export default DS.Model.extend({
   name: DS.attr("string"),
+  path: DS.attr("string"),
   children: DS.attr("array"),
   parent: DS.attr("string"),
   capacity: DS.attr("number"),
@@ -31,6 +32,10 @@ export default DS.Model.extend({
   absCapacity: DS.attr("number"),
   absMaxCapacity: DS.attr("number"),
   absUsedCapacity: DS.attr("number"),
+  weight: DS.attr("number"),
+  normalizedWeight: DS.attr("number"),
+  creationMethod: DS.attr("string"),
+  orderingPolicy: DS.attr("string"),
   state: DS.attr("string"),
   userLimit: DS.attr("number"),
   userLimitFactor: DS.attr("number"),
@@ -49,6 +54,14 @@ export default DS.Model.extend({
 return len <= 0;
   }.property("children"),
 
+  isWeightMode: function() {
+   return this.get("weight") !== -1;
+  }.property("children"),
+
+  isFlexibleDynamicQueue: function() {
+return this.get("creationMethod") === "dynamicFlexible";
+  }.property("children"),
+
   capacitiesBarChartData: function() {
 var floatToFixed = Converter.floatToFixed;
 return [
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
index 2dec520..e9ce25d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
@@ -92,6 +92,7 @@ export default DS.JSONAPISerializer.extend({
 type: primaryModelClass.modelName, // yarn-queue
 attributes: {
   name: payload.queueName,
+  path: payload.queuePath,
  

[hadoop] branch trunk updated: YARN-10636. CS Auto Queue creation should reject submissions with empty path parts. Contributed by Gergely Pollak.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new d28b6f9  YARN-10636. CS Auto Queue creation should reject submissions 
with empty path parts. Contributed by Gergely Pollak.
d28b6f9 is described below

commit d28b6f90c8c38f787d12cb01f5f254f2969046bc
Author: Peter Bacsko 
AuthorDate: Fri Feb 19 13:57:30 2021 +0100

YARN-10636. CS Auto Queue creation should reject submissions with empty 
path parts. Contributed by Gergely Pollak.
---
 .../scheduler/capacity/CapacityScheduler.java  | 15 ++-
 .../TestCapacitySchedulerAutoQueueCreation.java| 49 ++
 2 files changed, 62 insertions(+), 2 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 21d6248..36f831d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -962,6 +962,17 @@ public class CapacityScheduler extends
   if (placementContext == null) {
 fallbackContext = CSQueueUtils.extractQueuePath(queueName);
   }
+
+  //we need to make sure there is no empty path parts present
+  String path = fallbackContext.getFullQueuePath();
+  String[] pathParts = path.split("\\.");
+  for (int i = 0; i < pathParts.length; i++) {
+if ("".equals(pathParts[i])) {
+  LOG.error("Application submitted to invalid path: '{}'", path);
+  return null;
+}
+  }
+
   if (fallbackContext.hasParentQueue()) {
 try {
   return autoCreateLeafQueue(fallbackContext);
@@ -1022,8 +1033,8 @@ public class CapacityScheduler extends
   }
 
   //Could be a potential auto-created leaf queue
-  CSQueue queue = getOrCreateQueueFromPlacementContext(applicationId, user,
-queueName, placementContext, false);
+  CSQueue queue = getOrCreateQueueFromPlacementContext(
+   applicationId, user, queueName, placementContext, false);
 
   if (queue == null) {
 String message;
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
index 4dc0fab..be41a33 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
@@ -89,6 +90,7 @@ import static 
org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager
 .NO_LABEL;
 import static 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON;
 
+import static 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
@@ -535,6 +537,53 @@ public class TestCapacitySchedulerAutoQueueCreation
 }
   }
 
+
+  @Test
+  public void testAutoQueueCreationFailsForEmptyPathWithAQCAndWeightMode()
+  throws Exception {
+if (mockRM != null) {
+  mockRM.stop();
+}
+
+//We need a special configuration we only need a V2 queue auto creation
+//And weight mode, to allow dynamic auto queue creation for root
+CapacitySchedulerConfiguration conf = setupSchedulerConfiguration();
+conf.setAuto

[hadoop] branch trunk updated: YARN-10635. CSMapping rule can return paths with empty parts. Contributed by Gergely Pollak.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 4383726  YARN-10635. CSMapping rule can return paths with empty parts. 
Contributed by Gergely Pollak.
4383726 is described below

commit 4383726d194f7310c12d928aacd2de44e0711639
Author: Peter Bacsko 
AuthorDate: Fri Feb 19 12:01:31 2021 +0100

YARN-10635. CSMapping rule can return paths with empty parts. Contributed 
by Gergely Pollak.
---
 .../placement/CSMappingPlacementRule.java  | 33 +++--
 .../placement/MappingQueuePath.java| 15 ++
 .../placement/MappingRuleActions.java  |  6 ++-
 .../placement/TestCSMappingPlacementRule.java  | 55 --
 4 files changed, 89 insertions(+), 20 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
index 28d2cb6..821d055 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
@@ -194,6 +194,7 @@ public class CSMappingPlacementRule extends PlacementRule {
 String secondaryGroup = null;
 Iterator it = groupsSet.iterator();
 String primaryGroup = it.next();
+
 while (it.hasNext()) {
   String group = it.next();
   if (this.queueManager.getQueue(group) != null) {
@@ -203,8 +204,7 @@ public class CSMappingPlacementRule extends PlacementRule {
 }
 
 if (secondaryGroup == null && LOG.isDebugEnabled()) {
-  LOG.debug("User {} is not associated with any Secondary " +
-  "Group. Hence it may use the 'default' queue", user);
+  LOG.debug("User {} is not associated with any Secondary group", user);
 }
 
 vctx.put("%primary_group", primaryGroup);
@@ -223,7 +223,15 @@ public class CSMappingPlacementRule extends PlacementRule {
 //To place queues specifically to default, users must use root.default
 if (!asc.getQueue().equals(YarnConfiguration.DEFAULT_QUEUE_NAME)) {
   vctx.put("%specified", asc.getQueue());
+} else {
+  //Adding specified as empty will prevent it to be undefined and it won't
+  //try to place the application to a queue named '%specified', queue path
+  //validation will reject the empty path or the path with empty parts,
+  //so we sill still hit the fallback action of this rule if no queue
+  //is specified
+  vctx.put("%specified", "");
 }
+
 vctx.put("%application", asc.getApplicationName());
 vctx.put("%default", "root.default");
 try {
@@ -239,6 +247,12 @@ public class CSMappingPlacementRule extends PlacementRule {
   private String validateAndNormalizeQueue(
   String queueName, boolean allowCreate) throws YarnException {
 MappingQueuePath path = new MappingQueuePath(queueName);
+
+if (path.hasEmptyPart()) {
+  throw new YarnException("Invalid path returned by rule: '" +
+  queueName + "'");
+}
+
 String leaf = path.getLeafName();
 String parent = path.getParent();
 
@@ -335,14 +349,19 @@ public class CSMappingPlacementRule extends PlacementRule 
{
   MappingRule rule, VariableContext variables) {
 MappingRuleResult result = rule.evaluate(variables);
 
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Evaluated rule '{}' with result: '{}'", rule, result);
+}
+
 if (result.getResult() == MappingRuleResultType.PLACE) {
   try {
 result.updateNormalizedQueue(validateAndNormalizeQueue(
 result.getQueue(), result.isCreateAllowed()));
   } catch (Exception e) {
-LOG.info("Cannot place to queue '{}' returned by mapping rule. " +
-"Reason: {}", result.getQueue(), e.getMessage());
 result = rule.getFallback();
+LOG.info("Cannot place to queue '{}' returned by mapping rule. " +
+"Reason: '{}' Fallback operation: '{}'",
+result.getQueue(), e.getMessage(), result);
   }
 }
 
@@ -451,6 +470,12 @@ public class CSMappingPlacementRule extends PlacementRule {
   }
 }
 
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Placement final result '{}' fo

[hadoop] branch trunk updated: YARN-10600. Convert root queue in fs2cs weight mode conversion. Contributed by Benjamin Teke.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new fa15594  YARN-10600. Convert root queue in fs2cs weight mode 
conversion. Contributed by Benjamin Teke.
fa15594 is described below

commit fa15594ae6092249b506af81c2f9b2e1889fe04a
Author: Peter Bacsko 
AuthorDate: Thu Jan 28 20:58:51 2021 +0100

YARN-10600. Convert root queue in fs2cs weight mode conversion. Contributed 
by Benjamin Teke.
---
 .../weightconversion/WeightToWeightConverter.java  |  5 +
 .../weightconversion/TestWeightToWeightConverter.java  | 14 ++
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java
index 6dcd05c..cbc98e4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java
@@ -28,6 +28,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
 
 public class WeightToWeightConverter
 implements CapacityConverter {
+  private static final String ROOT_QUEUE = "root";
 
   @Override
   public void convertWeightsForChildQueues(FSQueue queue,
@@ -35,6 +36,10 @@ public class WeightToWeightConverter
 List children = queue.getChildQueues();
 
 if (queue instanceof FSParentQueue || !children.isEmpty()) {
+  if (queue.getName().equals(ROOT_QUEUE)) {
+csConfig.set(getProperty(queue), getWeightString(queue));
+  }
+
   children.forEach(fsQueue -> csConfig.set(
   getProperty(fsQueue), getWeightString(fsQueue)));
   csConfig.setBoolean(getAutoCreateV2EnabledProperty(queue), true);
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java
index 5f9b2d0..28eb2b6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java
@@ -44,7 +44,9 @@ public class TestWeightToWeightConverter extends 
WeightConverterTestBase {
 FSQueue root = createFSQueues();
 converter.convertWeightsForChildQueues(root, config);
 
-assertEquals("Converted items", 1,
+assertEquals("root weight", "1.0w",
+config.get(PREFIX + "root.capacity"));
+assertEquals("Converted items", 2,
 config.getPropsWithPrefix(PREFIX).size());
   }
 
@@ -53,9 +55,11 @@ public class TestWeightToWeightConverter extends 
WeightConverterTestBase {
 FSQueue root = createFSQueues(1);
 converter.convertWeightsForChildQueues(root, config);
 
+assertEquals("root weight", "1.0w",
+config.get(PREFIX + "root.capacity"));
 assertEquals("root.a weight", "1.0w",
 config.get(PREFIX + "root.a.capacity"));
-assertEquals("Number of properties", 2,
+assertEquals("Number of properties", 3,
 config.getPropsWithPrefix(PREFIX).size());
   }
 
@@ -65,8 +69,10 @@ public class TestWeightToWeightConverter extends 
WeightConverterTestBase {
 
 converter.convertWeightsForChildQueues(root, config);
 
-assertEquals("Number of properties", 4,
+assertEquals("Number of properties", 5,
 config.getPropsWithPrefix(PREFIX).size());
+assertEquals("root weight", "1.0w",
+config.get(PREFIX + "root.capacity"));
 asse

[hadoop] branch trunk updated: YARN-10581. CS Flexible Auto Queue Creation: Modify RM /scheduler endpoint to include queue creation type for queues. Contributed by Szilard Nemeth.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 17fdac8  YARN-10581. CS Flexible Auto Queue Creation: Modify RM 
/scheduler endpoint to include queue creation type for queues. Contributed by 
Szilard Nemeth.
17fdac8 is described below

commit 17fdac8de59970e4873e6e8ac1939c0edcdbfb85
Author: Peter Bacsko 
AuthorDate: Thu Jan 21 18:06:53 2021 +0100

YARN-10581. CS Flexible Auto Queue Creation: Modify RM /scheduler endpoint 
to include queue creation type for queues. Contributed by Szilard Nemeth.
---
 .../webapp/dao/CapacitySchedulerInfo.java  |   5 +
 .../webapp/dao/CapacitySchedulerQueueInfo.java |   6 +
 .../dao/helper/CapacitySchedulerInfoHelper.java|  26 +++
 .../webapp/TestRMWebServicesCapacitySched.java |   6 +-
 ...estRMWebServicesCapacitySchedDynamicConfig.java | 254 ++---
 .../TestRMWebServicesForCSWithPartitions.java  |   2 +-
 6 files changed, 265 insertions(+), 34 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index a6fcfe6..548cfe8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -57,6 +57,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
   protected int queuePriority;
   protected String orderingPolicyInfo;
   protected String mode;
+  protected String queueType;
 
   @XmlTransient
   static final float EPSILON = 1e-8f;
@@ -105,6 +106,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
   .getConfigName();
 }
 mode = CapacitySchedulerInfoHelper.getMode(parent);
+queueType = CapacitySchedulerInfoHelper.getQueueType(parent);
   }
 
   public float getCapacity() {
@@ -185,4 +187,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
 return mode;
   }
 
+  public String getQueueType() {
+return queueType;
+  }
 }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 5e45f53..dec666b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -90,6 +90,7 @@ public class CapacitySchedulerQueueInfo {
   protected boolean autoCreateChildQueueEnabled;
   protected LeafQueueTemplateInfo leafQueueTemplate;
   protected String mode;
+  protected String queueType;
 
   CapacitySchedulerQueueInfo() {
   };
@@ -135,6 +136,7 @@ public class CapacitySchedulerQueueInfo {
 populateQueueCapacities(qCapacities, qResQuotas);
 
 mode = CapacitySchedulerInfoHelper.getMode(q);
+queueType = CapacitySchedulerInfoHelper.getQueueType(q);
 
 ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
 populateQueueResourceUsage(queueResourceUsage);
@@ -319,6 +321,10 @@ public class CapacitySchedulerQueueInfo {
 return mode;
   }
 
+  public String getQueueType() {
+return queueType;
+  }
+
   public float getWeight() {
 return weight;
   }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java
index 9727f9f..52a5a5c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn

[hadoop] branch branch-3.1 updated: MAPREDUCE-7309. Improve performance of reading resource request for mapper/reducers from config. Contributed by Peter Bacsko & Wangda Tan.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.1 by this push:
 new 7d60217  MAPREDUCE-7309. Improve performance of reading resource 
request for mapper/reducers from config. Contributed by Peter Bacsko & Wangda 
Tan.
7d60217 is described below

commit 7d6021775489cc4c45e42e4419e794439deb7034
Author: Peter Bacsko 
AuthorDate: Wed Nov 25 11:46:00 2020 +0100

MAPREDUCE-7309. Improve performance of reading resource request for 
mapper/reducers from config. Contributed by Peter Bacsko & Wangda Tan.
---
 .../hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java| 12 +++-
 .../hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java|  7 +++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 6d1ac61..0007cdd 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -158,6 +158,9 @@ public abstract class TaskAttemptImpl implements
 org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt,
   EventHandler {
 
+  @VisibleForTesting
+  protected final static Map RESOURCE_REQUEST_CACHE
+  = new HashMap<>();
   static final Counters EMPTY_COUNTERS = new Counters();
   private static final Logger LOG =
   LoggerFactory.getLogger(TaskAttemptImpl.class);
@@ -172,7 +175,7 @@ public abstract class TaskAttemptImpl implements
   private final Clock clock;
   private final org.apache.hadoop.mapred.JobID oldJobId;
   private final TaskAttemptListener taskAttemptListener;
-  private final Resource resourceCapability;
+  private Resource resourceCapability;
   protected Set dataLocalHosts;
   protected Set dataLocalRacks;
   private final List diagnostics = new ArrayList();
@@ -707,6 +710,10 @@ public abstract class TaskAttemptImpl implements
 getResourceTypePrefix(taskType);
 boolean memorySet = false;
 boolean cpuVcoresSet = false;
+if (RESOURCE_REQUEST_CACHE.get(taskType) != null) {
+  resourceCapability = RESOURCE_REQUEST_CACHE.get(taskType);
+  return;
+}
 if (resourceTypePrefix != null) {
   List resourceRequests =
   ResourceUtils.getRequestedResourcesFromConfig(conf,
@@ -767,6 +774,9 @@ public abstract class TaskAttemptImpl implements
 if (!cpuVcoresSet) {
   this.resourceCapability.setVirtualCores(getCpuRequired(conf, taskType));
 }
+RESOURCE_REQUEST_CACHE.put(taskType, resourceCapability);
+LOG.info("Resource capability of task type {} is set to {}",
+taskType, resourceCapability);
   }
 
   private String getCpuVcoresKey(TaskType taskType) {
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index 947b1e9..14e2e36 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -42,6 +42,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptFailEvent;
 import org.junit.After;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -180,6 +181,11 @@ public class TestTaskAttempt{
 ResourceUtils.resetResourceTypes(new Configuration());
   }
 
+  @Before
+  public void before() {
+TaskAttemptImpl.RESOURCE_REQUEST_CACHE.clear();
+  }
+
   @After
   public void tearDown() {
 ResourceUtils.resetResourceTypes(new Configuration());
@@ -1634,6 +1640,7 @@ public class TestTaskAttempt{
   TestAppender testAppender = new TestAppender();
   final Logger logger = Logger.getLogger(TaskAttemptImpl.class);
   try {
+TaskAttemptImpl.RESOURCE_REQUEST_CACHE.clear();
 logger.addAppender(testAppender);
 EventHandler eventHandler = mock(EventHandler.clas

[hadoop] branch branch-3.2 updated: MAPREDUCE-7309. Improve performance of reading resource request for mapper/reducers from config. Contributed by Peter Bacsko & Wangda Tan.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new d727ab4  MAPREDUCE-7309. Improve performance of reading resource 
request for mapper/reducers from config. Contributed by Peter Bacsko & Wangda 
Tan.
d727ab4 is described below

commit d727ab4906cd495f3dbb321b091b0e3ba46202c2
Author: Peter Bacsko 
AuthorDate: Wed Nov 25 11:44:48 2020 +0100

MAPREDUCE-7309. Improve performance of reading resource request for 
mapper/reducers from config. Contributed by Peter Bacsko & Wangda Tan.
---
 .../hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java| 12 +++-
 .../hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java|  7 +++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 7fc35c7..0cddaf3 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -158,6 +158,9 @@ public abstract class TaskAttemptImpl implements
 org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt,
   EventHandler {
 
+  @VisibleForTesting
+  protected final static Map RESOURCE_REQUEST_CACHE
+  = new HashMap<>();
   static final Counters EMPTY_COUNTERS = new Counters();
   private static final Logger LOG =
   LoggerFactory.getLogger(TaskAttemptImpl.class);
@@ -172,7 +175,7 @@ public abstract class TaskAttemptImpl implements
   private final Clock clock;
   private final org.apache.hadoop.mapred.JobID oldJobId;
   private final TaskAttemptListener taskAttemptListener;
-  private final Resource resourceCapability;
+  private Resource resourceCapability;
   protected Set dataLocalHosts;
   protected Set dataLocalRacks;
   private final List diagnostics = new ArrayList();
@@ -707,6 +710,10 @@ public abstract class TaskAttemptImpl implements
 getResourceTypePrefix(taskType);
 boolean memorySet = false;
 boolean cpuVcoresSet = false;
+if (RESOURCE_REQUEST_CACHE.get(taskType) != null) {
+  resourceCapability = RESOURCE_REQUEST_CACHE.get(taskType);
+  return;
+}
 if (resourceTypePrefix != null) {
   List resourceRequests =
   ResourceUtils.getRequestedResourcesFromConfig(conf,
@@ -767,6 +774,9 @@ public abstract class TaskAttemptImpl implements
 if (!cpuVcoresSet) {
   this.resourceCapability.setVirtualCores(getCpuRequired(conf, taskType));
 }
+RESOURCE_REQUEST_CACHE.put(taskType, resourceCapability);
+LOG.info("Resource capability of task type {} is set to {}",
+taskType, resourceCapability);
   }
 
   private String getCpuVcoresKey(TaskType taskType) {
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index 7f54ea1..ba40849 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -42,6 +42,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
 import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptFailEvent;
 import org.junit.After;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -180,6 +181,11 @@ public class TestTaskAttempt{
 ResourceUtils.resetResourceTypes(new Configuration());
   }
 
+  @Before
+  public void before() {
+TaskAttemptImpl.RESOURCE_REQUEST_CACHE.clear();
+  }
+
   @After
   public void tearDown() {
 ResourceUtils.resetResourceTypes(new Configuration());
@@ -1634,6 +1640,7 @@ public class TestTaskAttempt{
   TestAppender testAppender = new TestAppender();
   final Logger logger = Logger.getLogger(TaskAttemptImpl.class);
   try {
+TaskAttemptImpl.RESOURCE_REQUEST_CACHE.clear();
 logger.addAppender(testAppender);
 EventHandler eventHandler = mock(EventHandler.clas

[hadoop] branch branch-3.3 updated: MAPREDUCE-7309. Improve performance of reading resource request for mapper/reducers from config. Contributed by Peter Bacsko & Wangda Tan.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 8ed5653  MAPREDUCE-7309. Improve performance of reading resource 
request for mapper/reducers from config. Contributed by Peter Bacsko & Wangda 
Tan.
8ed5653 is described below

commit 8ed565382f8dfa55e52c2512827e043ff13f5d20
Author: Peter Bacsko 
AuthorDate: Wed Nov 25 11:36:58 2020 +0100

MAPREDUCE-7309. Improve performance of reading resource request for 
mapper/reducers from config. Contributed by Peter Bacsko & Wangda Tan.
---
 .../hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java| 12 +++-
 .../hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java|  7 +++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 0f51fef..173ddcb 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -158,6 +158,9 @@ public abstract class TaskAttemptImpl implements
 org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt,
   EventHandler {
 
+  @VisibleForTesting
+  protected final static Map RESOURCE_REQUEST_CACHE
+  = new HashMap<>();
   static final Counters EMPTY_COUNTERS = new Counters();
   private static final Logger LOG =
   LoggerFactory.getLogger(TaskAttemptImpl.class);
@@ -172,7 +175,7 @@ public abstract class TaskAttemptImpl implements
   private final Clock clock;
   private final org.apache.hadoop.mapred.JobID oldJobId;
   private final TaskAttemptListener taskAttemptListener;
-  private final Resource resourceCapability;
+  private Resource resourceCapability;
   protected Set dataLocalHosts;
   protected Set dataLocalRacks;
   private final List diagnostics = new ArrayList();
@@ -707,6 +710,10 @@ public abstract class TaskAttemptImpl implements
 getResourceTypePrefix(taskType);
 boolean memorySet = false;
 boolean cpuVcoresSet = false;
+if (RESOURCE_REQUEST_CACHE.get(taskType) != null) {
+  resourceCapability = RESOURCE_REQUEST_CACHE.get(taskType);
+  return;
+}
 if (resourceTypePrefix != null) {
   List resourceRequests =
   ResourceUtils.getRequestedResourcesFromConfig(conf,
@@ -767,6 +774,9 @@ public abstract class TaskAttemptImpl implements
 if (!cpuVcoresSet) {
   this.resourceCapability.setVirtualCores(getCpuRequired(conf, taskType));
 }
+RESOURCE_REQUEST_CACHE.put(taskType, resourceCapability);
+LOG.info("Resource capability of task type {} is set to {}",
+taskType, resourceCapability);
   }
 
   private String getCpuVcoresKey(TaskType taskType) {
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index a5f0f7b..96d1ebe 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -43,6 +43,7 @@ import 
org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptFailEvent;
 import 
org.apache.hadoop.yarn.util.resource.CustomResourceTypesConfigurationProvider;
 import org.junit.After;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -155,6 +156,11 @@ public class TestTaskAttempt{
 ResourceUtils.resetResourceTypes(new Configuration());
   }
 
+  @Before
+  public void before() {
+TaskAttemptImpl.RESOURCE_REQUEST_CACHE.clear();
+  }
+
   @After
   public void tearDown() {
 ResourceUtils.resetResourceTypes(new Configuration());
@@ -1721,6 +1727,7 @@ public class TestTaskAttempt{
   TestAppender testAppender = new TestAppender();
   final Logger logger = Logger.getLogger(TaskAttemptImpl.class);
   try {
+TaskAttemptImpl.RESOURCE_REQUEST_CACHE.clear();
 logger.addAppender(testAppender);
 EventHan

[hadoop] branch branch-3.1 updated: MAPREDUCE-7304. Enhance the map-reduce Job end notifier to be able to notify the given URL via a custom class. Contributed by Zoltan Erdmann

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.1 by this push:
 new 6062978  MAPREDUCE-7304. Enhance the map-reduce Job end notifier to be 
able to notify the given URL via a custom class. Contributed by Zoltan Erdmann
6062978 is described below

commit 6062978768df51c3a531fa40fabf6b56d90a6e09
Author: Peter Bacsko 
AuthorDate: Fri Nov 20 13:16:48 2020 +0100

MAPREDUCE-7304. Enhance the map-reduce Job end notifier to be able to 
notify the given URL via a custom class. Contributed by Zoltan Erdmann
---
 .../hadoop/mapreduce/v2/app/JobEndNotifier.java| 51 ++-
 .../mapreduce/v2/app/TestJobEndNotifier.java   | 42 
 .../java/org/apache/hadoop/mapred/JobConf.java | 46 +
 .../hadoop/mapreduce/CustomJobEndNotifier.java | 57 ++
 .../org/apache/hadoop/mapreduce/MRJobConfig.java   |  3 ++
 .../src/main/resources/mapred-default.xml  | 17 +++
 6 files changed, 215 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
index 3bf0542..ed49f82 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
@@ -25,9 +25,11 @@ import java.net.MalformedURLException;
 import java.net.Proxy;
 import java.net.URL;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapreduce.CustomJobEndNotifier;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
 import org.eclipse.jetty.util.log.Log;
@@ -57,6 +59,9 @@ public class JobEndNotifier implements Configurable {
   protected int timeout; // Timeout (ms) on the connection and notification
   protected URL urlToNotify; //URL to notify read from the config
   protected Proxy proxyToUse = Proxy.NO_PROXY; //Proxy to use for notification
+  // A custom notifier implementation
+  // (see org.apache.hadoop.mapreduce.CustomJobEndNotifier)
+  private String customJobEndNotifierClassName;
 
   /**
* Parse the URL that needs to be notified of the end of the job, along
@@ -84,6 +89,9 @@ public class JobEndNotifier implements Configurable {
 
 proxyConf = conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY);
 
+customJobEndNotifierClassName = StringUtils.stripToNull(
+conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS));
+
 //Configure the proxy to use if its set. It should be set like
 //proxyType@proxyHostname:port
 if(proxyConf != null && !proxyConf.equals("") &&
@@ -115,11 +123,22 @@ public class JobEndNotifier implements Configurable {
   public Configuration getConf() {
 return conf;
   }
-  
+
   /**
* Notify the URL just once. Use best effort.
*/
   protected boolean notifyURLOnce() {
+if (customJobEndNotifierClassName == null) {
+  return notifyViaBuiltInNotifier();
+} else {
+  return notifyViaCustomNotifier();
+}
+  }
+
+  /**
+   * Uses a simple HttpURLConnection to do the Job end notification.
+   */
+  private boolean notifyViaBuiltInNotifier() {
 boolean success = false;
 try {
   Log.getLog().info("Job end notification trying " + urlToNotify);
@@ -146,6 +165,36 @@ public class JobEndNotifier implements Configurable {
   }
 
   /**
+   * Uses the custom Job end notifier class to do the Job end notification.
+   */
+  private boolean notifyViaCustomNotifier() {
+try {
+  Log.getLog().info("Will be using " + customJobEndNotifierClassName
++ " for Job end notification");
+
+  final Class customJobEndNotifierClass =
+  Class.forName(customJobEndNotifierClassName)
+  .asSubclass(CustomJobEndNotifier.class);
+  final CustomJobEndNotifier customJobEndNotifier =
+  customJobEndNotifierClass.getDeclaredConstructor().newInstance();
+
+  boolean success = customJobEndNotifier.notifyOnce(urlToNotify, conf);
+  if (success) {
+Log.getLog().info("Job end notification to " + urlToNotify
+  + " succeeded");
+  } else {
+Log.getLog().warn("Job end notification to "

[hadoop] branch branch-3.2 updated: MAPREDUCE-7304. Enhance the map-reduce Job end notifier to be able to notify the given URL via a custom class. Contributed by Zoltan Erdmann

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new 82db198  MAPREDUCE-7304. Enhance the map-reduce Job end notifier to be 
able to notify the given URL via a custom class. Contributed by Zoltan Erdmann
82db198 is described below

commit 82db198d24c7e47f5efaa4d7db6668b6544e539a
Author: Peter Bacsko 
AuthorDate: Fri Nov 20 13:15:47 2020 +0100

MAPREDUCE-7304. Enhance the map-reduce Job end notifier to be able to 
notify the given URL via a custom class. Contributed by Zoltan Erdmann
---
 .../hadoop/mapreduce/v2/app/JobEndNotifier.java| 51 ++-
 .../mapreduce/v2/app/TestJobEndNotifier.java   | 42 
 .../java/org/apache/hadoop/mapred/JobConf.java | 46 +
 .../hadoop/mapreduce/CustomJobEndNotifier.java | 57 ++
 .../org/apache/hadoop/mapreduce/MRJobConfig.java   |  3 ++
 .../src/main/resources/mapred-default.xml  | 17 +++
 6 files changed, 215 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
index 3bf0542..ed49f82 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
@@ -25,9 +25,11 @@ import java.net.MalformedURLException;
 import java.net.Proxy;
 import java.net.URL;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapreduce.CustomJobEndNotifier;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
 import org.eclipse.jetty.util.log.Log;
@@ -57,6 +59,9 @@ public class JobEndNotifier implements Configurable {
   protected int timeout; // Timeout (ms) on the connection and notification
   protected URL urlToNotify; //URL to notify read from the config
   protected Proxy proxyToUse = Proxy.NO_PROXY; //Proxy to use for notification
+  // A custom notifier implementation
+  // (see org.apache.hadoop.mapreduce.CustomJobEndNotifier)
+  private String customJobEndNotifierClassName;
 
   /**
* Parse the URL that needs to be notified of the end of the job, along
@@ -84,6 +89,9 @@ public class JobEndNotifier implements Configurable {
 
 proxyConf = conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY);
 
+customJobEndNotifierClassName = StringUtils.stripToNull(
+conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS));
+
 //Configure the proxy to use if its set. It should be set like
 //proxyType@proxyHostname:port
 if(proxyConf != null && !proxyConf.equals("") &&
@@ -115,11 +123,22 @@ public class JobEndNotifier implements Configurable {
   public Configuration getConf() {
 return conf;
   }
-  
+
   /**
* Notify the URL just once. Use best effort.
*/
   protected boolean notifyURLOnce() {
+if (customJobEndNotifierClassName == null) {
+  return notifyViaBuiltInNotifier();
+} else {
+  return notifyViaCustomNotifier();
+}
+  }
+
+  /**
+   * Uses a simple HttpURLConnection to do the Job end notification.
+   */
+  private boolean notifyViaBuiltInNotifier() {
 boolean success = false;
 try {
   Log.getLog().info("Job end notification trying " + urlToNotify);
@@ -146,6 +165,36 @@ public class JobEndNotifier implements Configurable {
   }
 
   /**
+   * Uses the custom Job end notifier class to do the Job end notification.
+   */
+  private boolean notifyViaCustomNotifier() {
+try {
+  Log.getLog().info("Will be using " + customJobEndNotifierClassName
++ " for Job end notification");
+
+  final Class customJobEndNotifierClass =
+  Class.forName(customJobEndNotifierClassName)
+  .asSubclass(CustomJobEndNotifier.class);
+  final CustomJobEndNotifier customJobEndNotifier =
+  customJobEndNotifierClass.getDeclaredConstructor().newInstance();
+
+  boolean success = customJobEndNotifier.notifyOnce(urlToNotify, conf);
+  if (success) {
+Log.getLog().info("Job end notification to " + urlToNotify
+  + " succeeded");
+  } else {
+Log.getLog().warn("Job end notification to "

[hadoop] branch branch-3.3 updated: MAPREDUCE-7304. Enhance the map-reduce Job end notifier to be able to notify the given URL via a custom class. Contributed by Zoltan Erdmann

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new ced08fd  MAPREDUCE-7304. Enhance the map-reduce Job end notifier to be 
able to notify the given URL via a custom class. Contributed by Zoltan Erdmann
ced08fd is described below

commit ced08fd87fd3d7218f91fd976bb8c807ac279b24
Author: Peter Bacsko 
AuthorDate: Fri Nov 20 13:14:49 2020 +0100

MAPREDUCE-7304. Enhance the map-reduce Job end notifier to be able to 
notify the given URL via a custom class. Contributed by Zoltan Erdmann
---
 .../hadoop/mapreduce/v2/app/JobEndNotifier.java| 51 ++-
 .../mapreduce/v2/app/TestJobEndNotifier.java   | 42 
 .../java/org/apache/hadoop/mapred/JobConf.java | 46 +
 .../hadoop/mapreduce/CustomJobEndNotifier.java | 57 ++
 .../org/apache/hadoop/mapreduce/MRJobConfig.java   |  3 ++
 .../src/main/resources/mapred-default.xml  | 17 +++
 6 files changed, 215 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
index 3bf0542..ed49f82 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
@@ -25,9 +25,11 @@ import java.net.MalformedURLException;
 import java.net.Proxy;
 import java.net.URL;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapreduce.CustomJobEndNotifier;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
 import org.eclipse.jetty.util.log.Log;
@@ -57,6 +59,9 @@ public class JobEndNotifier implements Configurable {
   protected int timeout; // Timeout (ms) on the connection and notification
   protected URL urlToNotify; //URL to notify read from the config
   protected Proxy proxyToUse = Proxy.NO_PROXY; //Proxy to use for notification
+  // A custom notifier implementation
+  // (see org.apache.hadoop.mapreduce.CustomJobEndNotifier)
+  private String customJobEndNotifierClassName;
 
   /**
* Parse the URL that needs to be notified of the end of the job, along
@@ -84,6 +89,9 @@ public class JobEndNotifier implements Configurable {
 
 proxyConf = conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY);
 
+customJobEndNotifierClassName = StringUtils.stripToNull(
+conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS));
+
 //Configure the proxy to use if its set. It should be set like
 //proxyType@proxyHostname:port
 if(proxyConf != null && !proxyConf.equals("") &&
@@ -115,11 +123,22 @@ public class JobEndNotifier implements Configurable {
   public Configuration getConf() {
 return conf;
   }
-  
+
   /**
* Notify the URL just once. Use best effort.
*/
   protected boolean notifyURLOnce() {
+if (customJobEndNotifierClassName == null) {
+  return notifyViaBuiltInNotifier();
+} else {
+  return notifyViaCustomNotifier();
+}
+  }
+
+  /**
+   * Uses a simple HttpURLConnection to do the Job end notification.
+   */
+  private boolean notifyViaBuiltInNotifier() {
 boolean success = false;
 try {
   Log.getLog().info("Job end notification trying " + urlToNotify);
@@ -146,6 +165,36 @@ public class JobEndNotifier implements Configurable {
   }
 
   /**
+   * Uses the custom Job end notifier class to do the Job end notification.
+   */
+  private boolean notifyViaCustomNotifier() {
+try {
+  Log.getLog().info("Will be using " + customJobEndNotifierClassName
++ " for Job end notification");
+
+  final Class customJobEndNotifierClass =
+  Class.forName(customJobEndNotifierClassName)
+  .asSubclass(CustomJobEndNotifier.class);
+  final CustomJobEndNotifier customJobEndNotifier =
+  customJobEndNotifierClass.getDeclaredConstructor().newInstance();
+
+  boolean success = customJobEndNotifier.notifyOnce(urlToNotify, conf);
+  if (success) {
+Log.getLog().info("Job end notification to " + urlToNotify
+  + " succeeded");
+  } else {
+Log.getLog().warn("Job end notification to "

[hadoop] branch trunk updated: MAPREDUCE-7304. Enhance the map-reduce Job end notifier to be able to notify the given URL via a custom class. Contributed by Zoltan Erdmann

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new fb92aa4  MAPREDUCE-7304. Enhance the map-reduce Job end notifier to be 
able to notify the given URL via a custom class. Contributed by Zoltan Erdmann
fb92aa4 is described below

commit fb92aa4012750c0ee297454feba1a562f8304a4c
Author: Peter Bacsko 
AuthorDate: Fri Nov 20 13:13:51 2020 +0100

MAPREDUCE-7304. Enhance the map-reduce Job end notifier to be able to 
notify the given URL via a custom class. Contributed by Zoltan Erdmann
---
 .../hadoop/mapreduce/v2/app/JobEndNotifier.java| 51 ++-
 .../mapreduce/v2/app/TestJobEndNotifier.java   | 42 
 .../java/org/apache/hadoop/mapred/JobConf.java | 46 +
 .../hadoop/mapreduce/CustomJobEndNotifier.java | 57 ++
 .../org/apache/hadoop/mapreduce/MRJobConfig.java   |  3 ++
 .../src/main/resources/mapred-default.xml  | 17 +++
 6 files changed, 215 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
index 3bf0542..ed49f82 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
@@ -25,9 +25,11 @@ import java.net.MalformedURLException;
 import java.net.Proxy;
 import java.net.URL;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapreduce.CustomJobEndNotifier;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
 import org.eclipse.jetty.util.log.Log;
@@ -57,6 +59,9 @@ public class JobEndNotifier implements Configurable {
   protected int timeout; // Timeout (ms) on the connection and notification
   protected URL urlToNotify; //URL to notify read from the config
   protected Proxy proxyToUse = Proxy.NO_PROXY; //Proxy to use for notification
+  // A custom notifier implementation
+  // (see org.apache.hadoop.mapreduce.CustomJobEndNotifier)
+  private String customJobEndNotifierClassName;
 
   /**
* Parse the URL that needs to be notified of the end of the job, along
@@ -84,6 +89,9 @@ public class JobEndNotifier implements Configurable {
 
 proxyConf = conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY);
 
+customJobEndNotifierClassName = StringUtils.stripToNull(
+conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_CUSTOM_NOTIFIER_CLASS));
+
 //Configure the proxy to use if its set. It should be set like
 //proxyType@proxyHostname:port
 if(proxyConf != null && !proxyConf.equals("") &&
@@ -115,11 +123,22 @@ public class JobEndNotifier implements Configurable {
   public Configuration getConf() {
 return conf;
   }
-  
+
   /**
* Notify the URL just once. Use best effort.
*/
   protected boolean notifyURLOnce() {
+if (customJobEndNotifierClassName == null) {
+  return notifyViaBuiltInNotifier();
+} else {
+  return notifyViaCustomNotifier();
+}
+  }
+
+  /**
+   * Uses a simple HttpURLConnection to do the Job end notification.
+   */
+  private boolean notifyViaBuiltInNotifier() {
 boolean success = false;
 try {
   Log.getLog().info("Job end notification trying " + urlToNotify);
@@ -146,6 +165,36 @@ public class JobEndNotifier implements Configurable {
   }
 
   /**
+   * Uses the custom Job end notifier class to do the Job end notification.
+   */
+  private boolean notifyViaCustomNotifier() {
+try {
+  Log.getLog().info("Will be using " + customJobEndNotifierClassName
++ " for Job end notification");
+
+  final Class customJobEndNotifierClass =
+  Class.forName(customJobEndNotifierClassName)
+  .asSubclass(CustomJobEndNotifier.class);
+  final CustomJobEndNotifier customJobEndNotifier =
+  customJobEndNotifierClass.getDeclaredConstructor().newInstance();
+
+  boolean success = customJobEndNotifier.notifyOnce(urlToNotify, conf);
+  if (success) {
+Log.getLog().info("Job end notification to " + urlToNotify
+  + " succeeded");
+  } else {
+Log.getLog().warn("Job end notification to "

[hadoop] branch branch-3.1 updated: YARN-10396. Max applications calculation per queue disregards queue level settings in absolute mode. Contributed by Benjamin Teke.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.1 by this push:
 new a86c91b  YARN-10396. Max applications calculation per queue disregards 
queue level settings in absolute mode. Contributed by Benjamin Teke.
a86c91b is described below

commit a86c91bc060662f7d4188d61a761123c2061e0b6
Author: Peter Bacsko 
AuthorDate: Mon Nov 16 11:54:19 2020 +0100

YARN-10396. Max applications calculation per queue disregards queue level 
settings in absolute mode. Contributed by Benjamin Teke.
---
 .../scheduler/capacity/ParentQueue.java|  14 ++-
 .../scheduler/capacity/TestParentQueue.java| 134 ++---
 2 files changed, 132 insertions(+), 16 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index e080d1c..2201728 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1114,8 +1114,18 @@ public class ParentQueue extends AbstractCSQueue {
 if (childQueue instanceof LeafQueue) {
   LeafQueue leafQueue = (LeafQueue) childQueue;
   CapacitySchedulerConfiguration conf = csContext.getConfiguration();
-  int maxApplications = (int) (conf.getMaximumSystemApplications()
-  * childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+  int maxApplications =
+  conf.getMaximumApplicationsPerQueue(childQueue.getQueuePath());
+  if (maxApplications < 0) {
+int maxGlobalPerQueueApps = 
conf.getGlobalMaximumApplicationsPerQueue();
+if (maxGlobalPerQueueApps > 0) {
+  maxApplications = (int) (maxGlobalPerQueueApps *
+  childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+} else {
+  maxApplications = (int) (conf.getMaximumSystemApplications()
+  * 
childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+}
+  }
   leafQueue.setMaxApplications(maxApplications);
 
   int maxApplicationsPerUser = Math.min(maxApplications,
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index fe66aba..6d04e29 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -109,15 +109,17 @@ public class TestParentQueue {
   
   private static final String A = "a";
   private static final String B = "b";
+  private static final String Q_A =
+  CapacitySchedulerConfiguration.ROOT + "." + A;
+  private static final String Q_B =
+  CapacitySchedulerConfiguration.ROOT + "." + B;
   private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) {
 
 // Define top-level queues
 conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
 
-final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
 conf.setCapacity(Q_A, 30);
 
-final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
 conf.setCapacity(Q_B, 70);
 
 LOG.info("Setup top-level queues a and b");
@@ -129,11 +131,9 @@ public class TestParentQueue {
 // Define top-level queues
 conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B});
 
-final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
 conf.setMinimumResourceRequirement("", Q_A,
 QUEUE_A_RESOURCE);
 
-final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
 conf.setMinimumResourceRequirement("", Q_B,
 QUEUE_B_RESOURCE);
 
@@ -369,9 +369,7 @@ public cl

[hadoop] branch branch-3.2 updated: YARN-10396. Max applications calculation per queue disregards queue level settings in absolute mode. Contributed by Benjamin Teke.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
 new c5ae78b  YARN-10396. Max applications calculation per queue disregards 
queue level settings in absolute mode. Contributed by Benjamin Teke.
c5ae78b is described below

commit c5ae78b7933d3fdb161714f396f95a54d6198360
Author: Peter Bacsko 
AuthorDate: Mon Nov 16 11:48:50 2020 +0100

YARN-10396. Max applications calculation per queue disregards queue level 
settings in absolute mode. Contributed by Benjamin Teke.
---
 .../scheduler/capacity/ParentQueue.java|  14 ++-
 .../scheduler/capacity/TestParentQueue.java| 134 ++---
 2 files changed, 132 insertions(+), 16 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 531e520..ee74f78 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1119,8 +1119,18 @@ public class ParentQueue extends AbstractCSQueue {
 if (childQueue instanceof LeafQueue) {
   LeafQueue leafQueue = (LeafQueue) childQueue;
   CapacitySchedulerConfiguration conf = csContext.getConfiguration();
-  int maxApplications = (int) (conf.getMaximumSystemApplications()
-  * childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+  int maxApplications =
+  conf.getMaximumApplicationsPerQueue(childQueue.getQueuePath());
+  if (maxApplications < 0) {
+int maxGlobalPerQueueApps = 
conf.getGlobalMaximumApplicationsPerQueue();
+if (maxGlobalPerQueueApps > 0) {
+  maxApplications = (int) (maxGlobalPerQueueApps *
+  childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+} else {
+  maxApplications = (int) (conf.getMaximumSystemApplications()
+  * 
childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+}
+  }
   leafQueue.setMaxApplications(maxApplications);
 
   int maxApplicationsPerUser = Math.min(maxApplications,
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 7a012a8..7fa1ee3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -110,15 +110,17 @@ public class TestParentQueue {
   
   private static final String A = "a";
   private static final String B = "b";
+  private static final String Q_A =
+  CapacitySchedulerConfiguration.ROOT + "." + A;
+  private static final String Q_B =
+  CapacitySchedulerConfiguration.ROOT + "." + B;
   private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) {
 
 // Define top-level queues
 conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
 
-final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
 conf.setCapacity(Q_A, 30);
 
-final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
 conf.setCapacity(Q_B, 70);
 
 LOG.info("Setup top-level queues a and b");
@@ -130,11 +132,9 @@ public class TestParentQueue {
 // Define top-level queues
 conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B});
 
-final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
 conf.setMinimumResourceRequirement("", Q_A,
 QUEUE_A_RESOURCE);
 
-final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
 conf.setMinimumResourceRequirement("", Q_B,
 QUEUE_B_RESOURCE);
 
@@ -370,9 +370,7 @@ public cl

[hadoop] 02/02: YARN-10425. Replace the legacy placement engine in CS with the new one. Contributed by Gergely Pollak.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit 567600fd80896c1c9b0db1f228368d4eb2a694a2
Author: Peter Bacsko 
AuthorDate: Wed Nov 11 17:10:15 2020 +0100

YARN-10425. Replace the legacy placement engine in CS with the new one. 
Contributed by Gergely Pollak.
---
 .../yarn/server/resourcemanager/RMAppManager.java  |   4 +-
 .../placement/AppNameMappingPlacementRule.java | 204 
 .../placement/CSMappingPlacementRule.java  |  92 +++-
 .../MappingRuleValidationContextImpl.java  |  16 +-
 .../placement/PlacementManager.java|  10 +-
 .../resourcemanager/placement/PlacementRule.java   |  25 +
 .../placement/UserGroupMappingPlacementRule.java   | 558 -
 .../scheduler/capacity/CapacityScheduler.java  |  44 +-
 .../capacity/CapacitySchedulerConfiguration.java   |  18 +
 .../server/resourcemanager/TestAppManager.java |   4 +-
 .../TestAppManagerWithFairScheduler.java   |  14 +-
 .../placement/TestPlacementManager.java|  31 +-
 .../TestUserGroupMappingPlacementRule.java |   4 +-
 .../TestAbsoluteResourceWithAutoQueue.java |   1 +
 .../TestCapacitySchedulerAutoCreatedQueueBase.java |  28 +-
 .../TestCapacitySchedulerAutoQueueCreation.java|  15 +-
 .../TestCapacitySchedulerQueueMappingFactory.java  |  43 +-
 .../scheduler/capacity/TestQueueMappings.java  |  29 +-
 18 files changed, 227 insertions(+), 913 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index fe18d82..13c2ec7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -864,9 +864,9 @@ public class RMAppManager implements 
EventHandler,
 if (placementManager != null) {
   try {
 String usernameUsedForPlacement =
-getUserNameForPlacement(user, context, placementManager);
+getUserNameForPlacement(user, context, placementManager);
 placementContext = placementManager
-.placeApplication(context, usernameUsedForPlacement);
+.placeApplication(context, usernameUsedForPlacement, isRecovery);
   } catch (YarnException e) {
 // Placement could also fail if the user doesn't exist in system
 // skip if the user is not found during recovery.
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
deleted file mode 100644
index 63d98ba..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/AppNameMappingPlacementRule.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.placement;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capac

[hadoop] branch trunk updated (b0ab222 -> 567600f)

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git.


from b0ab222  YARN-10425. Replace the legacy placement engine in CS with 
the new one. Contributed by Gergely Pollak.
 new cd0490e  Revert "YARN-10425. Replace the legacy placement engine in CS 
with the new one. Contributed by Gergely Pollak."
 new 567600f  YARN-10425. Replace the legacy placement engine in CS with 
the new one. Contributed by Gergely Pollak.

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../protocol/RefreshMountTableEntriesRequest.java  | 34 ++
 1 file changed, 34 insertions(+)


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] 01/02: Revert "YARN-10425. Replace the legacy placement engine in CS with the new one. Contributed by Gergely Pollak."

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit cd0490e8c6dd502c639de2d26980556b78eeeb06
Author: Peter Bacsko 
AuthorDate: Wed Nov 11 17:06:12 2020 +0100

Revert "YARN-10425. Replace the legacy placement engine in CS with the new 
one. Contributed by Gergely Pollak."

This reverts commit b0ab222a6c0073c31c285e89e43508b3e0ca9576.
---
 .../protocol/RefreshMountTableEntriesRequest.java  |  34 ++
 .../yarn/server/resourcemanager/RMAppManager.java  |   4 +-
 .../placement/AppNameMappingPlacementRule.java | 204 
 .../placement/CSMappingPlacementRule.java  |  92 +---
 .../MappingRuleValidationContextImpl.java  |  16 +-
 .../placement/PlacementManager.java|  10 +-
 .../resourcemanager/placement/PlacementRule.java   |  25 -
 .../placement/UserGroupMappingPlacementRule.java   | 558 +
 .../scheduler/capacity/CapacityScheduler.java  |  44 +-
 .../capacity/CapacitySchedulerConfiguration.java   |  18 -
 .../server/resourcemanager/TestAppManager.java |   4 +-
 .../TestAppManagerWithFairScheduler.java   |  14 +-
 .../placement/TestPlacementManager.java|  31 +-
 .../TestUserGroupMappingPlacementRule.java |   4 +-
 .../TestAbsoluteResourceWithAutoQueue.java |   1 -
 .../TestCapacitySchedulerAutoCreatedQueueBase.java |  28 +-
 .../TestCapacitySchedulerAutoQueueCreation.java|  15 +-
 .../TestCapacitySchedulerQueueMappingFactory.java  |  43 +-
 .../scheduler/capacity/TestQueueMappings.java  |  29 +-
 19 files changed, 947 insertions(+), 227 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshMountTableEntriesRequest.java
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshMountTableEntriesRequest.java
index e69de29..899afe7 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshMountTableEntriesRequest.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshMountTableEntriesRequest.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.store.protocol;
+
+import java.io.IOException;
+
+import 
org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
+
+/**
+ * API request for refreshing mount table cached entries from state store.
+ */
+public abstract class RefreshMountTableEntriesRequest {
+
+  public static RefreshMountTableEntriesRequest newInstance()
+  throws IOException {
+return StateStoreSerializer
+.newRecord(RefreshMountTableEntriesRequest.class);
+  }
+}
\ No newline at end of file
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 13c2ec7..fe18d82 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -864,9 +864,9 @@ public class RMAppManager implements 
EventHandler,
 if (placementManager != null) {
   try {
 String usernameUsedForPlacement =
-getUserNameForPlacement(user, context, placementManager);
+getUserNameForPlacement(user, context, placementManager);
 placementContext = placementManager
-.placeApplication(context, usernameUsedForPlacement, isRecovery);
+.placeApplication(context, usernameUsedForPlacement);
 

[hadoop] branch trunk updated: YARN-10425. Replace the legacy placement engine in CS with the new one. Contributed by Gergely Pollak.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new b0ab222  YARN-10425. Replace the legacy placement engine in CS with 
the new one. Contributed by Gergely Pollak.
b0ab222 is described below

commit b0ab222a6c0073c31c285e89e43508b3e0ca9576
Author: Peter Bacsko 
AuthorDate: Wed Nov 11 17:03:16 2020 +0100

YARN-10425. Replace the legacy placement engine in CS with the new one. 
Contributed by Gergely Pollak.
---
 .../protocol/RefreshMountTableEntriesRequest.java  |  34 --
 .../yarn/server/resourcemanager/RMAppManager.java  |   4 +-
 .../placement/AppNameMappingPlacementRule.java | 204 
 .../placement/CSMappingPlacementRule.java  |  92 +++-
 .../MappingRuleValidationContextImpl.java  |  16 +-
 .../placement/PlacementManager.java|  10 +-
 .../resourcemanager/placement/PlacementRule.java   |  25 +
 .../placement/UserGroupMappingPlacementRule.java   | 558 -
 .../scheduler/capacity/CapacityScheduler.java  |  44 +-
 .../capacity/CapacitySchedulerConfiguration.java   |  18 +
 .../server/resourcemanager/TestAppManager.java |   4 +-
 .../TestAppManagerWithFairScheduler.java   |  14 +-
 .../placement/TestPlacementManager.java|  31 +-
 .../TestUserGroupMappingPlacementRule.java |   4 +-
 .../TestAbsoluteResourceWithAutoQueue.java |   1 +
 .../TestCapacitySchedulerAutoCreatedQueueBase.java |  28 +-
 .../TestCapacitySchedulerAutoQueueCreation.java|  15 +-
 .../TestCapacitySchedulerQueueMappingFactory.java  |  43 +-
 .../scheduler/capacity/TestQueueMappings.java  |  29 +-
 19 files changed, 227 insertions(+), 947 deletions(-)

diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshMountTableEntriesRequest.java
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshMountTableEntriesRequest.java
index 899afe7..e69de29 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshMountTableEntriesRequest.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshMountTableEntriesRequest.java
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.federation.store.protocol;
-
-import java.io.IOException;
-
-import 
org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer;
-
-/**
- * API request for refreshing mount table cached entries from state store.
- */
-public abstract class RefreshMountTableEntriesRequest {
-
-  public static RefreshMountTableEntriesRequest newInstance()
-  throws IOException {
-return StateStoreSerializer
-.newRecord(RefreshMountTableEntriesRequest.class);
-  }
-}
\ No newline at end of file
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index fe18d82..13c2ec7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -864,9 +864,9 @@ public class RMAppManager implements 
EventHandler,
 if (placementManager != null) {
   try {
 String usernameUsedForPlacement =
-getUserNameForPlacement(user, context, placementManager);
+getUserNameForPlacement(user, context, placementManager);
 placementContext = placementManager
-.plac

[hadoop] branch branch-3.3 updated: YARN-10458. Hive On Tez queries fails upon submission to dynamically created pools. Contributed by Peter Bacsko.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.3 by this push:
 new 0361837  YARN-10458. Hive On Tez queries fails upon submission to 
dynamically created pools. Contributed by Peter Bacsko.
0361837 is described below

commit 0361837c9ed96e853d4e726ee7011abc19dfbbc2
Author: Peter Bacsko 
AuthorDate: Tue Nov 10 13:16:52 2020 +0100

YARN-10458. Hive On Tez queries fails upon submission to dynamically 
created pools. Contributed by Peter Bacsko.
---
 .../yarn/server/resourcemanager/RMAppManager.java  |  4 +
 .../scheduler/capacity/CapacityScheduler.java  | 20 +
 .../server/resourcemanager/TestAppManager.java | 94 +-
 .../TestCapacitySchedulerAutoQueueCreation.java| 64 ++-
 .../capacity/TestCapacitySchedulerQueueACLs.java   | 42 ++
 5 files changed, 219 insertions(+), 5 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 440b0ea..3c0da44 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -927,6 +927,10 @@ public class RMAppManager implements 
EventHandler,
 return usernameUsedForPlacement;
   }
   String queue = appPlacementContext.getQueue();
+  String parent = appPlacementContext.getParentQueue();
+  if (scheduler instanceof CapacityScheduler && parent != null) {
+queue = parent + "." + queue;
+  }
   if (callerUGI != null && scheduler
   .checkAccess(callerUGI, QueueACL.SUBMIT_APPLICATIONS, queue)) {
 usernameUsedForPlacement = userNameFromAppTag;
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index a540652..34fdaa0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2283,6 +2283,21 @@ public class CapacityScheduler extends
   public boolean checkAccess(UserGroupInformation callerUGI,
   QueueACL acl, String queueName) {
 CSQueue queue = getQueue(queueName);
+
+if (queueName.startsWith("root.")) {
+  // can only check proper ACLs if the path is fully qualified
+  while (queue == null) {
+int sepIndex = queueName.lastIndexOf(".");
+String parentName = queueName.substring(0, sepIndex);
+if (LOG.isDebugEnabled()) {
+  LOG.debug("Queue {} does not exist, checking parent {}",
+  queueName, parentName);
+}
+queueName = parentName;
+queue = queueManager.getQueue(queueName);
+  }
+}
+
 if (queue == null) {
   LOG.debug("ACL not found for queue access-type {} for queue {}",
   acl, queueName);
@@ -3294,4 +3309,9 @@ public class CapacityScheduler extends
   public boolean placementConstraintEnabled() {
 return true;
   }
+
+  @VisibleForTesting
+  public void setQueueManager(CapacitySchedulerQueueManager qm) {
+this.queueManager = qm;
+  }
 }
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 15a31bc..3ac6fee 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/ser

[hadoop] branch trunk updated: MAPREDUCE-7302. Upgrading to JUnit 4.13 causes testcase TestFetcher.testCorruptedIFile() to fail. Contributed by Peter Bacsko. Reviewed by Akira Ajisaka.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new da93cd9  MAPREDUCE-7302. Upgrading to JUnit 4.13 causes testcase 
TestFetcher.testCorruptedIFile() to fail. Contributed by Peter Bacsko. Reviewed 
by Akira Ajisaka.
da93cd9 is described below

commit da93cd962c8d19874d09360726628cdde8595a7d
Author: Peter Bacsko 
AuthorDate: Wed Oct 28 10:35:30 2020 +0100

MAPREDUCE-7302. Upgrading to JUnit 4.13 causes testcase 
TestFetcher.testCorruptedIFile() to fail. Contributed by Peter Bacsko. Reviewed 
by Akira Ajisaka.
---
 .../src/main/java/org/apache/hadoop/io/ReadaheadPool.java   | 13 -
 .../apache/hadoop/mapreduce/task/reduce/TestFetcher.java|  3 +++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
index 34ae2c6..b5acc35 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.io.nativeio.NativeIO;
 
 import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_WILLNEED;
 
+import 
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import 
org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.slf4j.Logger;
@@ -59,7 +60,17 @@ public class ReadaheadPool {
   return instance;
 }
   }
-  
+
+  @VisibleForTesting
+  public static void resetInstance() {
+synchronized (ReadaheadPool.class) {
+  if (instance != null) {
+instance.pool.shutdownNow();
+instance = null;
+  }
+}
+  }
+
   private ReadaheadPool() {
 pool = new ThreadPoolExecutor(POOL_SIZE, MAX_POOL_SIZE, 3L, 
TimeUnit.SECONDS,
 new ArrayBlockingQueue(CAPACITY));
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
index ed35ff6..7b420d3 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestFetcher.java
@@ -46,6 +46,7 @@ import java.util.ArrayList;
 
 import javax.crypto.SecretKey;
 
+import org.apache.hadoop.io.ReadaheadPool;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.Counters;
 import org.apache.hadoop.mapred.IFileInputStream;
@@ -92,6 +93,8 @@ public class TestFetcher {
   @SuppressWarnings("unchecked") // mocked generics
   public void setup() {
 LOG.info(">>>> " + name.getMethodName());
+// to avoid threading issues with JUnit 4.13+
+ReadaheadPool.resetInstance();
 job = new JobConf();
 job.setBoolean(MRJobConfig.SHUFFLE_FETCH_RETRY_ENABLED, false);
 jobWithRetry = new JobConf();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10424. Adapt existing AppName and UserGroupMapping unittests to ensure backwards compatibility. Contributed by Benjamin Teke.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new aba4a50  YARN-10424. Adapt existing AppName and UserGroupMapping 
unittests to ensure backwards compatibility. Contributed by Benjamin Teke.
aba4a50 is described below

commit aba4a506d64169238d3b3fd0b2b2afcd6c1796c3
Author: Peter Bacsko 
AuthorDate: Wed Sep 9 09:41:11 2020 +0200

YARN-10424. Adapt existing AppName and UserGroupMapping unittests to ensure 
backwards compatibility. Contributed by Benjamin Teke.
---
 .../placement/TestAppNameMappingPlacementRule.java | 102 ++---
 .../TestUserGroupMappingPlacementRule.java | 239 -
 2 files changed, 213 insertions(+), 128 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java
index 29141ae..9bcac9a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java
@@ -20,19 +20,24 @@ package 
org.apache.hadoop.yarn.server.resourcemanager.placement;
 
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.security.GroupMappingServiceProvider;
+import org.apache.hadoop.security.Groups;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SimpleGroupsMapping;
 import org.apache.hadoop.yarn.util.Records;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
-import org.mockito.Mockito;
 
-import java.util.Arrays;
+import java.io.IOException;
+import java.util.Collections;
 
+import static 
org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT;
+import static org.mockito.ArgumentMatchers.isNull;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -55,24 +60,37 @@ public class TestAppNameMappingPlacementRule {
 SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
   }
 
+  private void createQueueHierarchy(
+  CapacitySchedulerQueueManager queueManager) {
+MockQueueHierarchyBuilder.create()
+.withQueueManager(queueManager)
+.withQueue(ROOT_QUEUE + DOT + Q1_QUEUE)
+.withQueue(ROOT_QUEUE + DOT + Q2_QUEUE)
+.withQueue(ROOT_QUEUE + DOT + DEFAULT_QUEUE)
+.withQueue(ROOT_QUEUE + DOT + AMBIGUOUS_QUEUE)
+.withQueue(ROOT_QUEUE + DOT + USER_NAME + DOT + AMBIGUOUS_QUEUE)
+.build();
+
+when(queueManager.getQueue(isNull())).thenReturn(null);
+  }
+
   private void verifyQueueMapping(QueueMapping queueMapping,
-  String user, String expectedQueue) throws YarnException {
+  String user, String expectedQueue)
+  throws IOException, YarnException {
 verifyQueueMapping(queueMapping, user,
 queueMapping.getQueue(), expectedQueue, false);
   }
 
   private void verifyQueueMapping(QueueMapping queueMapping,
-  String user, String inputQueue, String expectedQueue,
-  boolean overwrite) throws YarnException {
-AppNameMappingPlacementRule rule = new AppNameMappingPlacementRule(
-overwrite, Arrays.asList(queueMapping));
+  String user, String inputQueue,
+  String expectedQueue, boolean overwrite)
+  throws IOException, YarnException {
+MappingRule rule = MappingRule.createLegacyRule(
+queueMapping.getType().toString(),
+queueMapping.getSource(),
+queueMapping.getFullPath());
 
-CapacitySchedulerQueueManager qm =
-mock(CapacitySchedulerQueueManager.class);
-when(qm.isAmbiguous(Mockito.isA(String.class))).thenReturn(false);
-when(qm.isAmbiguous(AMBIGUOUS_QUEUE)).thenReturn(true);
-
-rule.queueManager = qm;
+CSMappingPlacementRule engine

[hadoop] branch trunk updated: YARN-10415. Create a group matcher which checks ALL groups of the user. Contributed by Gergely Pollak.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new c4fb404  YARN-10415. Create a group matcher which checks ALL groups of 
the user. Contributed by Gergely Pollak.
c4fb404 is described below

commit c4fb4044b2caf211210c955d832146c8294aea69
Author: Peter Bacsko 
AuthorDate: Tue Sep 8 10:57:00 2020 +0200

YARN-10415. Create a group matcher which checks ALL groups of the user. 
Contributed by Gergely Pollak.
---
 .../placement/CSMappingPlacementRule.java  | 29 +-
 .../resourcemanager/placement/MappingRule.java |  2 +-
 .../placement/MappingRuleMatchers.java | 55 ---
 .../resourcemanager/placement/VariableContext.java | 32 ++-
 .../placement/TestCSMappingPlacementRule.java  | 33 
 .../resourcemanager/placement/TestMappingRule.java |  2 +
 .../placement/TestMappingRuleMatchers.java | 62 --
 .../placement/TestVariableContext.java | 29 ++
 8 files changed, 220 insertions(+), 24 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
index 2929ae0..9983ebc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
@@ -156,23 +156,24 @@ public class CSMappingPlacementRule extends PlacementRule 
{
 return mappingRules.size() > 0;
   }
 
-  private String getPrimaryGroup(String user) throws IOException {
-return groups.getGroupsSet(user).iterator().next();
-  }
-
   /**
-   * Traverse all secondary groups (as there could be more than one
-   * and position is not guaranteed) and ensure there is queue with
-   * the same name.
+   * Sets group related data for the provided variable context.
+   * Primary group is the first group returned by getGroups.
+   * To determine secondary group we traverse all groups
+   * (as there could be more than one and position is not guaranteed) and
+   * ensure there is queue with the same name.
+   * This method also sets the groups set for the variable context for group
+   * matching.
+   * @param vctx Variable context to be updated
* @param user Name of the user
-   * @return Name of the secondary group if found, null otherwise
* @throws IOException
*/
-  private String getSecondaryGroup(String user) throws IOException {
+  private void setupGroupsForVariableContext(VariableContext vctx, String user)
+  throws IOException {
 Set groupsSet = groups.getGroupsSet(user);
 String secondaryGroup = null;
 Iterator it = groupsSet.iterator();
-it.next();
+String primaryGroup = it.next();
 while (it.hasNext()) {
   String group = it.next();
   if (this.queueManager.getQueue(group) != null) {
@@ -185,7 +186,10 @@ public class CSMappingPlacementRule extends PlacementRule {
   LOG.debug("User {} is not associated with any Secondary " +
   "Group. Hence it may use the 'default' queue", user);
 }
-return secondaryGroup;
+
+vctx.put("%primary_group", primaryGroup);
+vctx.put("%secondary_group", secondaryGroup);
+vctx.putExtraDataset("groups", groupsSet);
   }
 
   private VariableContext createVariableContext(
@@ -195,9 +199,8 @@ public class CSMappingPlacementRule extends PlacementRule {
 vctx.put("%user", user);
 vctx.put("%specified", asc.getQueue());
 vctx.put("%application", asc.getApplicationName());
-vctx.put("%primary_group", getPrimaryGroup(user));
-vctx.put("%secondary_group", getSecondaryGroup(user));
 vctx.put("%default", "root.default");
+setupGroupsForVariableContext(vctx, user);
 
 vctx.setImmutables(immutableVariables);
 return vctx;
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java
index 50fb18f..e61ad95 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/

[hadoop] branch trunk updated: YARN-10411. Create an allowCreate flag for MappingRuleAction. Contributed by Gergely Pollak.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 360bbcd  YARN-10411. Create an allowCreate flag for MappingRuleAction. 
Contributed by Gergely Pollak.
360bbcd is described below

commit 360bbcd3bcb2f659b00eb677accfd909d6457e61
Author: Peter Bacsko 
AuthorDate: Mon Sep 7 11:39:03 2020 +0200

YARN-10411. Create an allowCreate flag for MappingRuleAction. Contributed 
by Gergely Pollak.
---
 .../placement/CSMappingPlacementRule.java  | 50 +-
 .../resourcemanager/placement/MappingRule.java |  3 +-
 .../placement/MappingRuleActions.java  | 22 +++---
 .../placement/MappingRuleResult.java   | 40 -
 .../placement/TestCSMappingPlacementRule.java  | 50 +++---
 .../resourcemanager/placement/TestMappingRule.java |  4 +-
 .../placement/TestMappingRuleActions.java  | 21 -
 7 files changed, 144 insertions(+), 46 deletions(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
index dc642cd..2929ae0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
@@ -203,15 +203,16 @@ public class CSMappingPlacementRule extends PlacementRule 
{
 return vctx;
   }
 
-  private String validateAndNormalizeQueue(String queueName)
-  throws YarnException {
+  private String validateAndNormalizeQueue(
+  String queueName, boolean allowCreate) throws YarnException {
 MappingQueuePath path = new MappingQueuePath(queueName);
 String leaf = path.getLeafName();
 String parent = path.getParent();
 
 String normalizedName;
 if (parent != null) {
-  normalizedName = validateAndNormalizeQueueWithParent(parent, leaf);
+  normalizedName = validateAndNormalizeQueueWithParent(
+  parent, leaf, allowCreate);
 } else {
   normalizedName = validateAndNormalizeQueueWithNoParent(leaf);
 }
@@ -225,8 +226,8 @@ public class CSMappingPlacementRule extends PlacementRule {
 return normalizedName;
   }
 
-  private String validateAndNormalizeQueueWithParent(String parent, String 
leaf)
-  throws YarnException {
+  private String validateAndNormalizeQueueWithParent(
+  String parent, String leaf, boolean allowCreate) throws YarnException {
 CSQueue parentQueue = queueManager.getQueue(parent);
 //we don't find the specified parent, so the placement rule is invalid
 //for this case
@@ -244,17 +245,26 @@ public class CSMappingPlacementRule extends PlacementRule 
{
 String parentPath = parentQueue.getQueuePath();
 String fullPath = parentPath + DOT + leaf;
 
-//if we have a parent which is not a managed parent, we check if the leaf
-//queue exists under this parent
-if (!(parentQueue instanceof ManagedParentQueue)) {
-  CSQueue queue = queueManager.getQueue(fullPath);
-  //if the queue doesn't exit we return null
-  if (queue == null) {
-throw new YarnException("Mapping rule specified a parent queue '" +
-parent + "', but it is not a managed parent queue, " +
-"and no queue exists with name '" + leaf + "' under it.");
-  }
+//checking if the queue actually exists
+CSQueue queue = queueManager.getQueue(fullPath);
+//if we have a parent which is not a managed parent and the queue doesn't
+//then it is an invalid target, since the queue won't be auto-created
+if (!(parentQueue instanceof ManagedParentQueue) && queue == null) {
+  throw new YarnException("Mapping rule specified a parent queue '" +
+  parent + "', but it is not a managed parent queue, " +
+  "and no queue exists with name '" + leaf + "' under it.");
 }
+
+//if the queue does not exist but the parent is managed we need to check if
+//auto-creation is allowed
+if (parentQueue instanceof ManagedParentQueue
+&& queue == null
+&& allowCreate == false) {
+  throw new YarnException("Mapping rule doesn't allow auto-creation of " +
+  "the queue '" + fullPath + "'");
+}

[hadoop] branch trunk updated: YARN-10376. Create a class that covers the functionality of UserGroupMappingPlacementRule and AppNameMappingPlacementRule using the new mapping rules. Contributed by Ger

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 1841a5b  YARN-10376. Create a class that covers the functionality of 
UserGroupMappingPlacementRule and AppNameMappingPlacementRule using the new 
mapping rules. Contributed by Gergely Pollak.
1841a5b is described below

commit 1841a5bb03f1745c8e8c3bf781a4c1c8fdadb772
Author: Peter Bacsko 
AuthorDate: Sat Sep 5 10:00:24 2020 +0200

YARN-10376. Create a class that covers the functionality of 
UserGroupMappingPlacementRule and AppNameMappingPlacementRule using the new 
mapping rules. Contributed by Gergely Pollak.
---
 .../apache/hadoop/yarn/conf/YarnConfiguration.java |   1 +
 .../placement/CSMappingPlacementRule.java  | 413 +
 .../placement/TestCSMappingPlacementRule.java  | 376 +++
 3 files changed, 790 insertions(+)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 2dbbfbf..7c78112 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1087,6 +1087,7 @@ public class YarnConfiguration extends Configuration {
   
   /** Default queue name */
   public static final String DEFAULT_QUEUE_NAME = "default";
+  public static final String DEFAULT_QUEUE_FULL_NAME = "root.default";
 
   /**
* Buckets (in minutes) for the number of apps running in each queue.
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
new file mode 100644
index 000..dc642cd
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
@@ -0,0 +1,413 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.placement;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.security.Groups;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import static 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
+
+/**
+ * This class is responsible for making application submissions to queue
+ * assignments, based on the configured ruleset. This class supports all
+ * features supported by UserGroupMappingPlacementRule and
+ * AppNameMappingPlacementRule classes, also adding some features which are
+ * present in fair scheduler queue placement. This helps to reduce the gap
+ * between the two schedulers.
+ */
+public class CSMappingPlacementRule extends PlacementRule {
+  private static final Logger LOG = LoggerFactory
+  .getLogger(CSMappingPlacementRule.class);
+
+  private CapacitySchedulerQueueManager queueManager;
+  private List mappingRules;
+
+  /**
+   * These are the variables we associate a special 

[hadoop] branch trunk updated: YARN-10375. CS Mapping rule config parser should return MappingRule objects. Contributed by Gergely Pollak.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 5c15815  YARN-10375. CS Mapping rule config parser should return 
MappingRule objects. Contributed by Gergely Pollak.
5c15815 is described below

commit 5c1581577393f6a14bf31db9b3fde61cf11a2608
Author: Peter Bacsko 
AuthorDate: Thu Sep 3 18:54:22 2020 +0200

YARN-10375. CS Mapping rule config parser should return MappingRule 
objects. Contributed by Gergely Pollak.
---
 .../capacity/CapacitySchedulerConfiguration.java   | 45 ++
 1 file changed, 45 insertions(+)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 96f7533..371c974 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -21,6 +21,7 @@ package 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
 import 
org.apache.hadoop.yarn.server.resourcemanager.placement.QueuePlacementRuleUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -280,6 +281,10 @@ public class CapacitySchedulerConfiguration extends 
ReservationSchedulerConfigur
   public static final String QUEUE_MAPPING = PREFIX + "queue-mappings";
 
   @Private
+  public static final String QUEUE_MAPPING_NAME =
+  YarnConfiguration.QUEUE_PLACEMENT_RULES + ".app-name";
+
+  @Private
   public static final String ENABLE_QUEUE_MAPPING_OVERRIDE = QUEUE_MAPPING + 
"-override.enable";
 
   @Private
@@ -1159,6 +1164,46 @@ public class CapacitySchedulerConfiguration extends 
ReservationSchedulerConfigur
 return mappings;
   }
 
+  public List getMappingRules() {
+List mappings = new ArrayList();
+Collection mappingsString =
+getTrimmedStringCollection(QUEUE_MAPPING);
+
+for (String mappingValue : mappingsString) {
+  String[] mapping =
+  StringUtils.getTrimmedStringCollection(mappingValue, ":")
+  .toArray(new String[] {});
+  if (mapping.length != 3 || mapping[1].length() == 0
+  || mapping[2].length() == 0) {
+throw new IllegalArgumentException(
+"Illegal queue mapping " + mappingValue);
+  }
+
+  if (mapping[0].equals("u") || mapping[0].equals("g")) {
+mappings.add(MappingRule.createLegacyRule(
+mapping[0], mapping[1], mapping[2]));
+  } else {
+throw new IllegalArgumentException(
+"unknown mapping prefix " + mapping[0]);
+  }
+}
+
+mappingsString = getTrimmedStringCollection(QUEUE_MAPPING_NAME);
+for (String mappingValue : mappingsString) {
+  String[] mapping =
+  StringUtils.getTrimmedStringCollection(mappingValue, ":")
+  .toArray(new String[] {});
+  if (mapping.length != 2 || mapping[1].length() == 0) {
+throw new IllegalArgumentException(
+"Illegal queue mapping " + mappingValue);
+  }
+
+  mappings.add(MappingRule.createLegacyRule(mapping[0], mapping[1]));
+}
+
+return mappings;
+  }
+
   @Private
   @VisibleForTesting
   public void setQueuePlacementRules(Collection queuePlacementRules) {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[hadoop] branch trunk updated: YARN-10372. Create MappingRule class to represent each CS mapping rule. Contributed by Gergely Pollak

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new 288dab7  YARN-10372. Create MappingRule class to represent each CS 
mapping rule. Contributed by Gergely Pollak
288dab7 is described below

commit 288dab75633b92f3104e61d3163e51d1f1263041
Author: Peter Bacsko 
AuthorDate: Thu Sep 3 15:02:48 2020 +0200

YARN-10372. Create MappingRule class to represent each CS mapping rule. 
Contributed by Gergely Pollak
---
 .../resourcemanager/placement/MappingRule.java | 136 
 .../resourcemanager/placement/TestMappingRule.java | 143 +
 2 files changed, 279 insertions(+)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java
new file mode 100644
index 000..e03be2a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingRule.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.placement;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * Mapping rule represents a single mapping setting defined by the user. All
+ * rules have matchers and actions. Matcher determine if a mapping rule applies
+ * to a given applicationSubmission, while action represent the course of 
action
+ * we need to take when a rule applies.
+ *
+ * MappingRules also support fallback actions, which will be evaluated when the
+ * main action fails due to any reason (Eg. trying to place to a queue which
+ * does not exist)
+ */
+public class MappingRule {
+  public static final String USER_MAPPING = "u";
+  public static final String GROUP_MAPPING = "g";
+  public static final String APPLICATION_MAPPING = "a";
+  private final MappingRuleMatcher matcher;
+  private final MappingRuleAction action;
+
+  public MappingRule(MappingRuleMatcher matcher, MappingRuleAction action) {
+this.matcher = matcher;
+this.action = action;
+  }
+
+  /**
+   * This method evaluates the rule, and returns the MappingRuleResult, if
+   * the rule matches, skip action otherwise.
+   * @param variables The variable context, which contains all the variables
+   * @return The rule's result or skip action if the rule doesn't apply
+   */
+  public MappingRuleResult evaluate(VariableContext variables) {
+if (matcher.match(variables)) {
+  return action.execute(variables);
+}
+
+return MappingRuleResult.createSkipResult();
+  }
+
+  /**
+   * Returns the associated action's fallback.
+   * @return The fallback of the action
+   */
+  public MappingRuleResult getFallback() {
+return action.getFallback();
+  }
+
+  /**
+   * Creates a MappingRule object from the legacy style configuration. The
+   * configuration is a [TYPE]:SOURCE:PATH (eg. u:bob:root.users.%user).
+   * Using the source and path parts of the legacy rule, this method will
+   * create an application MappingRule which behaves as the legacy rule 
defined.
+   * This version of the method does not require type, since legacy application
+   * mappings omitted the 'a', so this method is to be used for those rules,
+   * which in all case are application mappings.
+   * @param source This part of the rule determines which applications the rule
+   *   will be applied
+   * @param path The path where the application is to be placed
+   * @return MappingRule based on the provided settings
+   */
+  public static MappingRule createLegacyRule(String source, String path) {
+return createLegacyRule(APPLICATION_MAPPING, source, path);
+  }
+
+  /**
+   * Creates a MappingRule object from the legacy style

[hadoop] branch trunk updated: YARN-10374. Create Actions for CS mapping rules. Contributed by Gergely Pollak.

[pbacsko]

This is an automated email from the ASF dual-hosted git repository.

pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
 new c7dab2b  YARN-10374. Create Actions for CS mapping rules. Contributed 
by Gergely Pollak.
c7dab2b is described below

commit c7dab2b23e41a3668c5d47a8ffdbf680ba64fd2d
Author: Peter Bacsko 
AuthorDate: Tue Sep 1 17:38:49 2020 +0200

YARN-10374. Create Actions for CS mapping rules. Contributed by Gergely 
Pollak.
---
 .../placement/MappingQueuePath.java| 108 +
 .../placement/MappingRuleAction.java   |  77 +++
 .../placement/MappingRuleActionBase.java   |  82 +++
 .../placement/MappingRuleActions.java  | 236 +++
 .../placement/MappingRuleResult.java   | 165 ++
 .../placement/MappingRuleResultType.java   |  43 
 .../placement/MappingRuleValidationContext.java|  75 ++
 .../MappingRuleValidationContextImpl.java  | 252 +
 .../placement/MockQueueHierarchyBuilder.java   |  12 +-
 .../placement/TestMappingRuleActions.java  | 173 ++
 .../TestMappingRuleValidationContextImpl.java  | 216 ++
 11 files changed, 1438 insertions(+), 1 deletion(-)

diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingQueuePath.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingQueuePath.java
new file mode 100644
index 000..2c01821
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MappingQueuePath.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.placement;
+
+import static 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
+
+/**
+ * This is a helper class which represents a queue path, and has easy access
+ * methods to get the path's parent or leaf part, or as a whole.
+ */
+public class MappingQueuePath {
+  /**
+   * The parent part of the queue path.
+   */
+  private String parent;
+
+  /**
+   * The leaf part of the parent path.
+   */
+  private String leaf;
+
+  /**
+   * Constructor to create mapping queue path from parent path and leaf name.
+   * @param parent Parent path of the queue
+   * @param leaf Name of the leaf queue
+   */
+  public MappingQueuePath(String parent, String leaf) {
+this.parent = parent;
+this.leaf = leaf;
+  }
+
+  /**
+   * Constructor creates a MappingQueuePath object using the queue's full path.
+   * @param fullPath Full path of the queue
+   */
+  public MappingQueuePath(String fullPath) {
+setFromFullPath(fullPath);
+  }
+
+  /**
+   * This method is responsible for splitting up a full queue path into parent
+   * path and leaf name.
+   * @param fullPath Full path of the queue to be processed
+   */
+  private void setFromFullPath(String fullPath) {
+parent = null;
+leaf = fullPath;
+
+int lastDotIdx = fullPath.lastIndexOf(DOT);
+if (lastDotIdx > -1) {
+  parent = fullPath.substring(0, lastDotIdx).trim();
+  leaf = fullPath.substring(lastDotIdx + 1).trim();
+}
+  }
+
+  /**
+   * Getter for the parent part of the path.
+   * @return Parent path of the queue, null if there is no parent.
+   */
+  public String getParent() {
+return parent;
+  }
+
+  /**
+   * Getter for the leaf part of the path.
+   * @return The leaf queue name
+   */
+  public String getLeafName() {
+return leaf;
+  }
+
+  /**
+   * Getter for the full path of the queue.
+   * @return Full path of the queue
+   */
+  public String getFullPath() {
+return hasParent() ? (parent + DOT + leaf) : leaf;
+  }
+
+  /**
+   * Convenience getter to check if the