[
https://issues.apache.org/jira/browse/YARN-5607?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17636496#comment-17636496
]
ASF GitHub Bot commented on YARN-5607:
--------------------------------------
K0K0V0K commented on code in PR #5119:
URL: https://github.com/apache/hadoop/pull/5119#discussion_r1027655301
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/CommonUtil.java:
##########
@@ -0,0 +1,410 @@
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public class CommonUtil {
+ static final Logger LOG = LoggerFactory.getLogger(MockRM.class);
+ private static final int SECOND = 1000;
+ private static final int TIMEOUT_MS_FOR_ATTEMPT = 40 * SECOND;
+ private static final int TIMEOUT_MS_FOR_APP_REMOVED = 40 * SECOND;
+ private static final int TIMEOUT_MS_FOR_CONTAINER_AND_NODE = 20 * SECOND;
+ private static final int WAIT_MS_PER_LOOP = 10;
+
+ /**
+ * Wait until an application has reached a specified state.
+ * The timeout is 80 seconds.
+ *
+ * @param appId the id of an application
+ * @param finalState the application state waited
+ * @throws InterruptedException if interrupted while waiting for the state
transition
+ */
+ public static void waitForState(MockRM rm, ApplicationId appId, RMAppState
finalState)
+ throws InterruptedException {
+ rm.drainEventsImplicitly();
+ RMApp app = rm.getRMContext().getRMApps().get(appId);
+ Assert.assertNotNull("app shouldn't be null", app);
+ final int timeoutMsecs = 80 * SECOND;
+ int timeWaiting = 0;
+ while (!finalState.equals(app.getState())) {
+ if (timeWaiting >= timeoutMsecs) {
+ break;
+ }
+
+ LOG.info("App : " + appId + " State is : " + app.getState() +
+ " Waiting for state : " + finalState);
+ Thread.sleep(WAIT_MS_PER_LOOP);
+ timeWaiting += WAIT_MS_PER_LOOP;
+ }
+
+ LOG.info("App State is : " + app.getState());
+ Assert.assertEquals("App State is not correct (timeout).", finalState,
+ app.getState());
+ }
+
+ /**
+ * Wait until an attempt has reached a specified state.
+ * The timeout is 40 seconds.
+ *
+ * @param attemptId the id of an attempt
+ * @param finalState the attempt state waited
+ * @throws InterruptedException if interrupted while waiting for the state
transition
+ */
+ public static void waitForState(MockRM rm, ApplicationAttemptId attemptId,
+ RMAppAttemptState finalState) throws
InterruptedException {
+ waitForState(rm, attemptId, finalState, TIMEOUT_MS_FOR_ATTEMPT);
+ }
+
+ /**
+ * Wait until an attempt has reached a specified state.
+ * The timeout can be specified by the parameter.
+ *
+ * @param attemptId the id of an attempt
+ * @param finalState the attempt state waited
+ * @param timeoutMsecs the length of timeout in milliseconds
+ * @throws InterruptedException if interrupted while waiting for the state
transition
+ */
+ private static void waitForState(MockRM rm, ApplicationAttemptId attemptId,
+ RMAppAttemptState finalState, int timeoutMsecs)
+ throws InterruptedException {
+ rm.start();
+ rm.drainEventsImplicitly();
+ RMApp app =
rm.getRMContext().getRMApps().get(attemptId.getApplicationId());
+ Assert.assertNotNull("app shouldn't be null", app);
+ RMAppAttempt attempt = app.getRMAppAttempt(attemptId);
+ waitForState(attempt, finalState, timeoutMsecs);
+ }
+
+ /**
+ * Wait until an attempt has reached a specified state.
+ * The timeout is 40 seconds.
+ *
+ * @param attempt an attempt
+ * @param finalState the attempt state waited
+ * @throws InterruptedException if interrupted while waiting for the state
transition
+ */
+ public static void waitForState(RMAppAttempt attempt,
+ RMAppAttemptState finalState) throws
InterruptedException {
+ waitForState(attempt, finalState, TIMEOUT_MS_FOR_ATTEMPT);
+ }
+
+ /**
+ * Wait until an attempt has reached a specified state.
+ * The timeout can be specified by the parameter.
+ *
+ * @param attempt an attempt
+ * @param finalState the attempt state waited
+ * @param timeoutMsecs the length of timeout in milliseconds
+ * @throws InterruptedException if interrupted while waiting for the state
transition
+ */
+ public static void waitForState(RMAppAttempt attempt,
+ RMAppAttemptState finalState, int
timeoutMsecs)
+ throws InterruptedException {
+ int timeWaiting = 0;
+ while (finalState != attempt.getAppAttemptState()) {
+ if (timeWaiting >= timeoutMsecs) {
+ break;
+ }
+
+ LOG.info("AppAttempt : " + attempt.getAppAttemptId() + " State is
: " +
+ attempt.getAppAttemptState() + " Waiting for state : " +
finalState);
+ Thread.sleep(WAIT_MS_PER_LOOP);
+ timeWaiting += WAIT_MS_PER_LOOP;
+ }
+
+ LOG.info("Attempt State is : " + attempt.getAppAttemptState());
+ Assert.assertEquals("Attempt state is not correct (timeout).",
finalState,
+ attempt.getState());
+ }
+
+ public static void waitForContainerToComplete(MockRM rm, RMAppAttempt
attempt,
+ NMContainerStatus
completedContainer) throws InterruptedException {
+ rm.drainEventsImplicitly();
+ int timeWaiting = 0;
+ while (timeWaiting < TIMEOUT_MS_FOR_CONTAINER_AND_NODE) {
+ List<ContainerStatus> containers =
attempt.getJustFinishedContainers();
+ LOG.info("Received completed containers " + containers);
+ for (ContainerStatus container : containers) {
+ if (container.getContainerId().equals(
+ completedContainer.getContainerId())) {
+ return;
+ }
+ }
+ Thread.sleep(WAIT_MS_PER_LOOP);
+ timeWaiting += WAIT_MS_PER_LOOP;
+ }
+ }
+
+ public static MockAM waitForNewAMToLaunchAndRegister(MockRM rm,
ApplicationId appId, int attemptSize,
+ MockNM nm) throws Exception {
+ RMApp app = rm.getRMContext().getRMApps().get(appId);
+ Assert.assertNotNull(app);
+ int timeWaiting = 0;
+ while (app.getAppAttempts().size() != attemptSize) {
+ if (timeWaiting >= TIMEOUT_MS_FOR_ATTEMPT) {
+ break;
+ }
+ LOG.info("Application " + appId
+ + " is waiting for AM to restart. Current has "
+ + app.getAppAttempts().size() + " attempts.");
+ Thread.sleep(WAIT_MS_PER_LOOP);
+ timeWaiting += WAIT_MS_PER_LOOP;
+ }
+ return rm.launchAndRegisterAM(app, rm, nm);
+ }
+
+ /**
+ * Wait until a container has reached a specified state.
+ * The timeout is 10 seconds.
Review Comment:
So i mean in 2017 the value of the **TIMEOUT_MS_FOR_CONTAINER_AND_NODE** was
increased from 10 to 20 with the YARN-5876 ticket, but the author forget to
update this javadoc.
To prevent these scenarios we can use the @link annotation instead of
hardcoded values.
> Document TestContainerResourceUsage#waitForContainerCompletion
> --------------------------------------------------------------
>
> Key: YARN-5607
> URL: https://issues.apache.org/jira/browse/YARN-5607
> Project: Hadoop YARN
> Issue Type: Test
> Components: resourcemanager, test
> Affects Versions: 2.9.0
> Reporter: Karthik Kambatla
> Assignee: Susheel Gupta
> Priority: Major
> Labels: newbie, pull-request-available
>
> The logic in TestContainerResourceUsage#waitForContainerCompletion
> (introduced in YARN-5024) is not immediately obvious. It could use some
> documentation. Also, this seems like a useful helper method. Should this be
> moved to one of the mock classes or to a util class?
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
