[hadoop] branch trunk updated: YARN-10672. All testcases in TestReservations are flaky. Contributed By Szilard Nemeth.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c6b30a5 YARN-10672. All testcases in TestReservations are flaky.
Contributed By Szilard Nemeth.
c6b30a5 is described below
commit c6b30a59dd20db39f34db8ce61ec152d7a90dad6
Author: Peter Bacsko
AuthorDate: Fri Mar 5 23:13:35 2021 +0100
YARN-10672. All testcases in TestReservations are flaky. Contributed By
Szilard Nemeth.
---
.../server/resourcemanager/scheduler/capacity/TestReservations.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index a9cee2d..1168f64 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -105,7 +105,6 @@ public class TestReservations {
CapacityScheduler spyCs = new CapacityScheduler();
cs = spy(spyCs);
rmContext = TestUtils.getMockRMContext();
-
}
private void setup(CapacitySchedulerConfiguration csConf) throws Exception {
@@ -114,6 +113,9 @@ public class TestReservations {
private void setup(CapacitySchedulerConfiguration csConf,
boolean addUserLimits) throws Exception {
+//All stub calls on the spy object of the 'cs' field should happen
+//before cs.start() is invoked. See YARN-10672 for more details.
+when(cs.getNumClusterNodes()).thenReturn(3);
csConf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS,
true);
final String newRoot = "root" + System.currentTimeMillis();
@@ -156,8 +158,6 @@ public class TestReservations {
cs.setRMContext(spyRMContext);
cs.init(csConf);
cs.start();
-
-when(cs.getNumClusterNodes()).thenReturn(3);
}
private static final String A = "a";
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-10664. Allow parameter expansion in NM_ADMIN_USER_ENV. Contributed by Jim Brennan.
This is an automated email from the ASF dual-hosted git repository.
ebadger pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 1a33878 YARN-10664. Allow parameter expansion in NM_ADMIN_USER_ENV.
Contributed by Jim Brennan.
1a33878 is described below
commit 1a3387827908547245a46a6a4edb1ce4a067e0a1
Author: Eric Badger
AuthorDate: Fri Mar 5 19:49:49 2021 +
YARN-10664. Allow parameter expansion in NM_ADMIN_USER_ENV. Contributed by
Jim
Brennan.
(cherry picked from commit 9d088639bfe1556669f39f01816d6f1703b77f4f)
---
.../containermanager/launcher/ContainerLaunch.java | 66 +--
.../launcher/TestContainerLaunch.java | 97 +-
2 files changed, 136 insertions(+), 27 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 4ea7909..e864c14 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -179,15 +179,36 @@ public class ContainerLaunch implements Callable
{
return var;
}
- private Map expandAllEnvironmentVars(
- ContainerLaunchContext launchContext, Path containerLogDir) {
-Map environment = launchContext.getEnvironment();
+ private void expandAllEnvironmentVars(
+ Map environment, Path containerLogDir) {
for (Entry entry : environment.entrySet()) {
String value = entry.getValue();
value = expandEnvironment(value, containerLogDir);
entry.setValue(value);
}
-return environment;
+ }
+
+ private void addKeystoreVars(Map environment,
+ Path containerWorkDir) {
+environment.put(ApplicationConstants.KEYSTORE_FILE_LOCATION_ENV_NAME,
+new Path(containerWorkDir,
+ContainerLaunch.KEYSTORE_FILE).toUri().getPath());
+environment.put(ApplicationConstants.KEYSTORE_PASSWORD_ENV_NAME,
+new String(container.getCredentials().getSecretKey(
+AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE_PASSWORD),
+StandardCharsets.UTF_8));
+ }
+
+ private void addTruststoreVars(Map environment,
+ Path containerWorkDir) {
+environment.put(
+ApplicationConstants.TRUSTSTORE_FILE_LOCATION_ENV_NAME,
+new Path(containerWorkDir,
+ContainerLaunch.TRUSTSTORE_FILE).toUri().getPath());
+environment.put(ApplicationConstants.TRUSTSTORE_PASSWORD_ENV_NAME,
+new String(container.getCredentials().getSecretKey(
+AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE_PASSWORD),
+StandardCharsets.UTF_8));
}
@Override
@@ -222,8 +243,10 @@ public class ContainerLaunch implements Callable {
}
launchContext.setCommands(newCmds);
- Map environment = expandAllEnvironmentVars(
- launchContext, containerLogDir);
+ // The actual expansion of environment variables happens after calling
+ // sanitizeEnv. This allows variables specified in NM_ADMIN_USER_ENV
+ // to reference user or container-defined variables.
+ Map environment = launchContext.getEnvironment();
// /// End of variable expansion
// Use this to track variables that are added to the environment by nm.
@@ -289,13 +312,6 @@ public class ContainerLaunch implements Callable {
lfs.create(nmPrivateKeystorePath,
EnumSet.of(CREATE, OVERWRITE))) {
keystoreOutStream.write(keystore);
- environment.put(ApplicationConstants.KEYSTORE_FILE_LOCATION_ENV_NAME,
- new Path(containerWorkDir,
- ContainerLaunch.KEYSTORE_FILE).toUri().getPath());
- environment.put(ApplicationConstants.KEYSTORE_PASSWORD_ENV_NAME,
- new String(container.getCredentials().getSecretKey(
- AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE_PASSWORD),
- StandardCharsets.UTF_8));
}
} else {
nmPrivateKeystorePath = null;
@@ -307,14 +323,6 @@ public class ContainerLaunch implements Callable {
lfs.create(nmPrivateTruststorePath,
EnumSet.of(CREATE, OVERWRITE))) {
truststoreOutStream.write(truststore);
- environment.put(
- ApplicationConstants.TRUSTSTO
[hadoop] branch trunk updated: YARN-10664. Allow parameter expansion in NM_ADMIN_USER_ENV. Contributed by Jim Brennan.
This is an automated email from the ASF dual-hosted git repository.
ebadger pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9d08863 YARN-10664. Allow parameter expansion in NM_ADMIN_USER_ENV.
Contributed by Jim Brennan.
9d08863 is described below
commit 9d088639bfe1556669f39f01816d6f1703b77f4f
Author: Eric Badger
AuthorDate: Fri Mar 5 19:46:40 2021 +
YARN-10664. Allow parameter expansion in NM_ADMIN_USER_ENV. Contributed by
Jim
Brennan.
---
.../containermanager/launcher/ContainerLaunch.java | 66 +--
.../launcher/TestContainerLaunch.java | 97 +-
2 files changed, 136 insertions(+), 27 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 4ea7909..e864c14 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -179,15 +179,36 @@ public class ContainerLaunch implements Callable
{
return var;
}
- private Map expandAllEnvironmentVars(
- ContainerLaunchContext launchContext, Path containerLogDir) {
-Map environment = launchContext.getEnvironment();
+ private void expandAllEnvironmentVars(
+ Map environment, Path containerLogDir) {
for (Entry entry : environment.entrySet()) {
String value = entry.getValue();
value = expandEnvironment(value, containerLogDir);
entry.setValue(value);
}
-return environment;
+ }
+
+ private void addKeystoreVars(Map environment,
+ Path containerWorkDir) {
+environment.put(ApplicationConstants.KEYSTORE_FILE_LOCATION_ENV_NAME,
+new Path(containerWorkDir,
+ContainerLaunch.KEYSTORE_FILE).toUri().getPath());
+environment.put(ApplicationConstants.KEYSTORE_PASSWORD_ENV_NAME,
+new String(container.getCredentials().getSecretKey(
+AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE_PASSWORD),
+StandardCharsets.UTF_8));
+ }
+
+ private void addTruststoreVars(Map environment,
+ Path containerWorkDir) {
+environment.put(
+ApplicationConstants.TRUSTSTORE_FILE_LOCATION_ENV_NAME,
+new Path(containerWorkDir,
+ContainerLaunch.TRUSTSTORE_FILE).toUri().getPath());
+environment.put(ApplicationConstants.TRUSTSTORE_PASSWORD_ENV_NAME,
+new String(container.getCredentials().getSecretKey(
+AMSecretKeys.YARN_APPLICATION_AM_TRUSTSTORE_PASSWORD),
+StandardCharsets.UTF_8));
}
@Override
@@ -222,8 +243,10 @@ public class ContainerLaunch implements Callable {
}
launchContext.setCommands(newCmds);
- Map environment = expandAllEnvironmentVars(
- launchContext, containerLogDir);
+ // The actual expansion of environment variables happens after calling
+ // sanitizeEnv. This allows variables specified in NM_ADMIN_USER_ENV
+ // to reference user or container-defined variables.
+ Map environment = launchContext.getEnvironment();
// /// End of variable expansion
// Use this to track variables that are added to the environment by nm.
@@ -289,13 +312,6 @@ public class ContainerLaunch implements Callable {
lfs.create(nmPrivateKeystorePath,
EnumSet.of(CREATE, OVERWRITE))) {
keystoreOutStream.write(keystore);
- environment.put(ApplicationConstants.KEYSTORE_FILE_LOCATION_ENV_NAME,
- new Path(containerWorkDir,
- ContainerLaunch.KEYSTORE_FILE).toUri().getPath());
- environment.put(ApplicationConstants.KEYSTORE_PASSWORD_ENV_NAME,
- new String(container.getCredentials().getSecretKey(
- AMSecretKeys.YARN_APPLICATION_AM_KEYSTORE_PASSWORD),
- StandardCharsets.UTF_8));
}
} else {
nmPrivateKeystorePath = null;
@@ -307,14 +323,6 @@ public class ContainerLaunch implements Callable {
lfs.create(nmPrivateTruststorePath,
EnumSet.of(CREATE, OVERWRITE))) {
truststoreOutStream.write(truststore);
- environment.put(
- ApplicationConstants.TRUSTSTORE_FILE_LOCATION_ENV_NAME,
- new Path(containerWorkDir,
-
[hadoop] branch trunk updated: YARN-10640. Adjust the queue Configured capacity to Configured weight number for weight mode in UI. Contributed by Qi Zhu.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e82e7c5 YARN-10640. Adjust the queue Configured capacity to
Configured weight number for weight mode in UI. Contributed by Qi Zhu.
e82e7c5 is described below
commit e82e7c597a130bb7a4e4b3132be85b7099961a2d
Author: Peter Bacsko
AuthorDate: Fri Mar 5 15:56:56 2021 +0100
YARN-10640. Adjust the queue Configured capacity to Configured weight
number for weight mode in UI. Contributed by Qi Zhu.
---
.../yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 47c888d..8212a72 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -159,8 +159,12 @@ class CapacitySchedulerPage extends RmView {
__("Used Capacity:",
appendPercent(resourceUsages.getUsed(),
capacities.getUsedCapacity() / 100))
- .__("Configured Capacity:",
- capacities.getConfiguredMinResource() == null ?
+ .__(capacities.getWeight() != -1 ?
+ "Configured Weight:" :
+ "Configured Capacity:",
+ capacities.getWeight() != -1 ?
+ capacities.getWeight() :
+ capacities.getConfiguredMinResource() == null ?
Resources.none().toString() :
capacities.getConfiguredMinResource().toString())
.__("Configured Max Capacity:",
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: HADOOP-17563. Update Bouncy Castle to 1.68. (#2740)
This is an automated email from the ASF dual-hosted git repository. tasanuma pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 7a8542b HADOOP-17563. Update Bouncy Castle to 1.68. (#2740) 7a8542b is described below commit 7a8542b833172ba4c78c8a504d2b8b401e18b6f2 Author: Takanobu Asanuma AuthorDate: Fri Mar 5 22:56:51 2021 +0900 HADOOP-17563. Update Bouncy Castle to 1.68. (#2740) (cherry picked from commit 077411675679900f94adba5329d0e33a4a528793) --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 7833145..bdc3955 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -98,7 +98,7 @@ 4.0 2.9.9 -1.60 +1.68 2.0.0-M21 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: HADOOP-17563. Update Bouncy Castle to 1.68. (#2740)
This is an automated email from the ASF dual-hosted git repository. tasanuma pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new e607d03 HADOOP-17563. Update Bouncy Castle to 1.68. (#2740) e607d03 is described below commit e607d039954a63b78e8ae98513999527502201fa Author: Takanobu Asanuma AuthorDate: Fri Mar 5 22:56:51 2021 +0900 HADOOP-17563. Update Bouncy Castle to 1.68. (#2740) (cherry picked from commit 077411675679900f94adba5329d0e33a4a528793) --- LICENSE-binary | 4 ++-- hadoop-project/pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 15e6805..9cb43fe 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -468,8 +468,8 @@ com.microsoft.azure:azure-cosmosdb-gateway:2.4.5 com.microsoft.azure:azure-data-lake-store-sdk:2.3.3 com.microsoft.azure:azure-keyvault-core:1.0.0 com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7 -org.bouncycastle:bcpkix-jdk15on:1.60 -org.bouncycastle:bcprov-jdk15on:1.60 +org.bouncycastle:bcpkix-jdk15on:1.68 +org.bouncycastle:bcprov-jdk15on:1.68 org.checkerframework:checker-qual:2.5.2 org.codehaus.mojo:animal-sniffer-annotations:1.17 org.jruby.jcodings:jcodings:1.0.13 diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index fd13bd2..afbd60a 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -108,7 +108,7 @@ 4.0 2.9.9 -1.60 +1.68 2.0.0-M21 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-17563. Update Bouncy Castle to 1.68. (#2740)
This is an automated email from the ASF dual-hosted git repository. tasanuma pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 0774116 HADOOP-17563. Update Bouncy Castle to 1.68. (#2740) 0774116 is described below commit 077411675679900f94adba5329d0e33a4a528793 Author: Takanobu Asanuma AuthorDate: Fri Mar 5 22:56:51 2021 +0900 HADOOP-17563. Update Bouncy Castle to 1.68. (#2740) --- LICENSE-binary | 4 ++-- hadoop-project/pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index 4a4b953..9053b60 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -468,8 +468,8 @@ com.microsoft.azure:azure-cosmosdb-gateway:2.4.5 com.microsoft.azure:azure-data-lake-store-sdk:2.3.3 com.microsoft.azure:azure-keyvault-core:1.0.0 com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7 -org.bouncycastle:bcpkix-jdk15on:1.60 -org.bouncycastle:bcprov-jdk15on:1.60 +org.bouncycastle:bcpkix-jdk15on:1.68 +org.bouncycastle:bcprov-jdk15on:1.68 org.checkerframework:checker-qual:2.5.2 org.codehaus.mojo:animal-sniffer-annotations:1.17 org.jruby.jcodings:jcodings:1.0.13 diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index c26ceac..7c74615 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -107,7 +107,7 @@ 27.0-jre 4.2.3 -1.60 +1.68 2.0.0-M21 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10642. Race condition: AsyncDispatcher can get stuck by the changes introduced in YARN-8995. Contributed by zhengchenyu.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e7cad38 YARN-10642. Race condition: AsyncDispatcher can get stuck by
the changes introduced in YARN-8995. Contributed by zhengchenyu.
e7cad38 is described below
commit e7cad3811fd14ff0b963423a24a9012ce5fb07a9
Author: Peter Bacsko
AuthorDate: Fri Mar 5 13:50:45 2021 +0100
YARN-10642. Race condition: AsyncDispatcher can get stuck by the changes
introduced in YARN-8995. Contributed by zhengchenyu.
---
.../apache/hadoop/yarn/event/AsyncDispatcher.java | 19 ---
.../hadoop/yarn/event/TestAsyncDispatcher.java | 58 +-
2 files changed, 68 insertions(+), 9 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 79ad464..f9deab0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -20,11 +20,11 @@ package org.apache.hadoop.yarn.event;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
-import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -268,11 +268,16 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
}
class GenericEventHandler implements EventHandler {
-private void printEventQueueDetails(BlockingQueue queue) {
- Map counterMap = eventQueue.stream().
- collect(Collectors.
- groupingBy(e -> e.getType(), Collectors.counting())
- );
+private void printEventQueueDetails() {
+ Iterator iterator = eventQueue.iterator();
+ Map counterMap = new HashMap<>();
+ while (iterator.hasNext()) {
+Enum eventType = iterator.next().getType();
+if (!counterMap.containsKey(eventType)) {
+ counterMap.put(eventType, 0L);
+}
+counterMap.put(eventType, counterMap.get(eventType) + 1);
+ }
for (Map.Entry entry : counterMap.entrySet()) {
long num = entry.getValue();
LOG.info("Event type: " + entry.getKey()
@@ -295,7 +300,7 @@ public class AsyncDispatcher extends AbstractService
implements Dispatcher {
if (qSize != 0 && qSize % detailsInterval == 0
&& lastEventDetailsQueueSizeLogged != qSize) {
lastEventDetailsQueueSizeLogged = qSize;
-printEventQueueDetails(eventQueue);
+printEventQueueDetails();
printTrigger = true;
}
int remCapacity = eventQueue.remainingCapacity();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
index 762e228..55ddd12 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
@@ -97,12 +97,23 @@ public class TestAsyncDispatcher {
}
private static class TestHandler implements EventHandler {
+
+private long sleepTime = 1500;
+
+TestHandler() {
+}
+
+TestHandler(long sleepTime) {
+ this.sleepTime = sleepTime;
+}
+
@Override
public void handle(Event event) {
try {
// As long as 1 events queued
-Thread.sleep(1500);
- } catch (InterruptedException e) {}
+Thread.sleep(this.sleepTime);
+ } catch (InterruptedException e) {
+ }
}
}
@@ -170,11 +181,54 @@ public class TestAsyncDispatcher {
//Make sure more than one event to take
verify(log, atLeastOnce()).
info("Latest dispatch event type: TestEventType");
+} finally {
+ //... restore logger object
+ logger.set(null, oldLog);
dispatcher.stop();
+}
+ }
+
+ //Test print dispatcher details when the blocking queue is heavy
+ @Test(timeout = 6)
+ public void testPrintDispatcherEventDetailsAvoidDeadLoop() throws Exception {
+for (int i = 0; i < 5; i++) {
+ testPrintDispatcherEventDetailsAvoidDeadLoopInternal();
+}
+ }
+
+ public void testPrintDispatcherEventDetailsAvoidDeadLoopInternal()
+ throws Exception {
+YarnConfiguration conf = new Yarn
[hadoop] branch trunk updated: YARN-10639. Queueinfo related capacity, should adjusted to weight mode. Contributed by Qi Zhu.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e19c009 YARN-10639. Queueinfo related capacity, should adjusted to
weight mode. Contributed by Qi Zhu.
e19c009 is described below
commit e19c00925f38346dc1291cdced108ecc0206fb74
Author: Peter Bacsko
AuthorDate: Fri Mar 5 13:18:06 2021 +0100
YARN-10639. Queueinfo related capacity, should adjusted to weight mode.
Contributed by Qi Zhu.
---
.../apache/hadoop/yarn/api/records/QueueInfo.java | 23 ---
.../src/main/proto/yarn_protos.proto | 1 +
.../apache/hadoop/yarn/client/cli/QueueCLI.java| 2 +
.../hadoop/yarn/client/ProtocolHATestBase.java | 5 ++-
.../apache/hadoop/yarn/client/cli/TestYarnCLI.java | 8 +++-
.../yarn/api/records/impl/pb/QueueInfoPBImpl.java | 12 ++
.../apache/hadoop/yarn/api/TestPBImplRecords.java | 2 +-
.../scheduler/capacity/AbstractCSQueue.java| 1 +
.../TestConfigurationMutationACLPolicies.java | 2 +-
.../scheduler/TestSchedulerApplicationAttempt.java | 2 +-
.../capacity/TestCapacitySchedulerWeightMode.java | 45 +++---
.../scheduler/capacity/TestLeafQueue.java | 2 +-
12 files changed, 87 insertions(+), 18 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
index 57ea9bf..803adad 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
@@ -59,7 +59,7 @@ public abstract class QueueInfo {
List childQueues, List applications,
QueueState queueState, Set accessibleNodeLabels,
String defaultNodeLabelExpression, QueueStatistics queueStatistics,
- boolean preemptionDisabled) {
+ boolean preemptionDisabled, float weight) {
QueueInfo queueInfo = Records.newRecord(QueueInfo.class);
queueInfo.setQueueName(queueName);
queueInfo.setCapacity(capacity);
@@ -72,6 +72,7 @@ public abstract class QueueInfo {
queueInfo.setDefaultNodeLabelExpression(defaultNodeLabelExpression);
queueInfo.setQueueStatistics(queueStatistics);
queueInfo.setPreemptionDisabled(preemptionDisabled);
+queueInfo.setWeight(weight);
return queueInfo;
}
@@ -82,14 +83,14 @@ public abstract class QueueInfo {
List childQueues, List applications,
QueueState queueState, Set accessibleNodeLabels,
String defaultNodeLabelExpression, QueueStatistics queueStatistics,
- boolean preemptionDisabled,
+ boolean preemptionDisabled, float weight,
Map queueConfigurations) {
QueueInfo queueInfo = QueueInfo.newInstance(queueName, capacity,
maximumCapacity, currentCapacity,
childQueues, applications,
queueState, accessibleNodeLabels,
defaultNodeLabelExpression, queueStatistics,
-preemptionDisabled);
+preemptionDisabled, weight);
queueInfo.setQueueConfigurations(queueConfigurations);
return queueInfo;
}
@@ -101,7 +102,7 @@ public abstract class QueueInfo {
List childQueues, List applications,
QueueState queueState, Set accessibleNodeLabels,
String defaultNodeLabelExpression, QueueStatistics queueStatistics,
- boolean preemptionDisabled,
+ boolean preemptionDisabled, float weight,
Map queueConfigurations,
boolean intraQueuePreemptionDisabled) {
QueueInfo queueInfo = QueueInfo.newInstance(queueName, capacity,
@@ -109,7 +110,7 @@ public abstract class QueueInfo {
childQueues, applications,
queueState, accessibleNodeLabels,
defaultNodeLabelExpression, queueStatistics,
-preemptionDisabled, queueConfigurations);
+preemptionDisabled, weight, queueConfigurations);
queueInfo.setIntraQueuePreemptionDisabled(intraQueuePreemptionDisabled);
return queueInfo;
}
@@ -137,6 +138,18 @@ public abstract class QueueInfo {
@Private
@Unstable
public abstract void setCapacity(float capacity);
+
+ /**
+ * Get the configured weight of the queue.
+ * @return configured weight of the queue
+ */
+ @Public
+ @Stable
+ public abstract float getWeight();
+
+ @Private
+ @Unstable
+ public abstract void setWeight(float weight);
/**
* Get the maximum capacity of the queue.
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index d7ca2a5..467f26a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/
