Repository: hadoop Updated Branches: refs/heads/yarn-native-services f1a358e17 -> 164c0c4c9
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java new file mode 100644 index 0000000..1a1593a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; + +/** + * A {@link CharSequence} appender that considers its {@link #limit} as upper + * bound. + * <p> + * When {@link #limit} would be reached on append, past messages will be + * truncated from head, and a header telling the user about truncation will be + * prepended, with ellipses in between header and messages. + * <p> + * Note that header and ellipses are not counted against {@link #limit}. + * <p> + * An example: + * + * <pre> + * {@code + * // At the beginning it's an empty string + * final Appendable shortAppender = new BoundedAppender(80); + * // The whole message fits into limit + * shortAppender.append( + * "message1 this is a very long message but fitting into limit\n"); + * // The first message is truncated, the second not + * shortAppender.append("message2 this is shorter than the previous one\n"); + * // The first message is deleted, the second truncated, the third + * // preserved + * shortAppender.append("message3 this is even shorter message, maybe.\n"); + * // The first two are deleted, the third one truncated, the last preserved + * shortAppender.append("message4 the shortest one, yet the greatest :)"); + * // Current contents are like this: + * // Diagnostic messages truncated, showing last 80 chars out of 199: + * // ...s is even shorter message, maybe. + * // message4 the shortest one, yet the greatest :) + * } + * </pre> + * <p> + * Note that <tt>null</tt> values are {@link #append(CharSequence) append}ed + * just like in {@link StringBuilder#append(CharSequence) original + * implementation}. + * <p> + * Note that this class is not thread safe. + */ +@VisibleForTesting +public class BoundedAppender { + @VisibleForTesting + public static final String TRUNCATED_MESSAGES_TEMPLATE = + "Diagnostic messages truncated, showing last " + + "%d chars out of %d:%n...%s"; + + private final int limit; + private final StringBuilder messages = new StringBuilder(); + private int totalCharacterCount = 0; + + public BoundedAppender(final int limit) { + Preconditions.checkArgument(limit > 0, "limit should be positive"); + + this.limit = limit; + } + + /** + * Append a {@link CharSequence} considering {@link #limit}, truncating + * from the head of {@code csq} or {@link #messages} when necessary. + * + * @param csq the {@link CharSequence} to append + * @return this + */ + public BoundedAppender append(final CharSequence csq) { + appendAndCount(csq); + checkAndCut(); + + return this; + } + + private void appendAndCount(final CharSequence csq) { + final int before = messages.length(); + messages.append(csq); + final int after = messages.length(); + totalCharacterCount += after - before; + } + + private void checkAndCut() { + if (messages.length() > limit) { + final int newStart = messages.length() - limit; + messages.delete(0, newStart); + } + } + + /** + * Get current length of messages considering truncates + * without header and ellipses. + * + * @return current length + */ + public int length() { + return messages.length(); + } + + /** + * Get a string representation of the actual contents, displaying also a + * header and ellipses when there was a truncate. + * + * @return String representation of the {@link #messages} + */ + @Override + public String toString() { + if (messages.length() < totalCharacterCount) { + return String.format(TRUNCATED_MESSAGES_TEMPLATE, messages.length(), + totalCharacterCount, messages.toString()); + } + + return messages.toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java new file mode 100644 index 0000000..2b9cfce --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.junit.Assert.assertEquals; + +/** + * Test class for {@link BoundedAppender}. + */ +public class TestBoundedAppender { + @Rule + public ExpectedException expected = ExpectedException.none(); + + @Test + public void initWithZeroLimitThrowsException() { + expected.expect(IllegalArgumentException.class); + expected.expectMessage("limit should be positive"); + + new BoundedAppender(0); + } + + @Test + public void nullAppendedNullStringRead() { + final BoundedAppender boundedAppender = new BoundedAppender(4); + boundedAppender.append(null); + + assertEquals("null appended, \"null\" read", "null", + boundedAppender.toString()); + } + + @Test + public void appendBelowLimitOnceValueIsReadCorrectly() { + final BoundedAppender boundedAppender = new BoundedAppender(2); + + boundedAppender.append("ab"); + + assertEquals("value appended is read correctly", "ab", + boundedAppender.toString()); + } + + @Test + public void appendValuesBelowLimitAreReadCorrectlyInFifoOrder() { + final BoundedAppender boundedAppender = new BoundedAppender(3); + + boundedAppender.append("ab"); + boundedAppender.append("cd"); + boundedAppender.append("e"); + boundedAppender.append("fg"); + + assertEquals("last values appended fitting limit are read correctly", + String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 7, "efg"), + boundedAppender.toString()); + } + + @Test + public void appendLastAboveLimitPreservesLastMessagePostfix() { + final BoundedAppender boundedAppender = new BoundedAppender(3); + + boundedAppender.append("ab"); + boundedAppender.append("cde"); + boundedAppender.append("fghij"); + + assertEquals( + "last value appended above limit postfix is read correctly", String + .format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 10, "hij"), + boundedAppender.toString()); + } + + @Test + public void appendMiddleAboveLimitPreservesLastMessageAndMiddlePostfix() { + final BoundedAppender boundedAppender = new BoundedAppender(3); + + boundedAppender.append("ab"); + boundedAppender.append("cde"); + + assertEquals("last value appended above limit postfix is read correctly", + String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 5, "cde"), + boundedAppender.toString()); + + boundedAppender.append("fg"); + + assertEquals( + "middle value appended above limit postfix and last value are " + + "read correctly", + String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 7, "efg"), + boundedAppender.toString()); + + boundedAppender.append("hijkl"); + + assertEquals( + "last value appended above limit postfix is read correctly", String + .format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 12, "jkl"), + boundedAppender.toString()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java index 443654d..704b097 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java @@ -67,6 +67,14 @@ public final class RegistryOperationsFactory { return operations; } + public static RegistryOperationsClient createClient(String name, + Configuration conf) { + Preconditions.checkArgument(conf != null, "Null configuration"); + RegistryOperationsClient operations = new RegistryOperationsClient(name); + operations.init(conf); + return operations; + } + /** * Create and initialize an anonymous read/write registry operations instance. * In a secure cluster, this instance will only have read access to the http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java index ad008c4..8713920 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java @@ -261,10 +261,9 @@ public class CuratorService extends CompositeService int retryCeiling = conf.getInt(KEY_REGISTRY_ZK_RETRY_CEILING, DEFAULT_ZK_RETRY_CEILING); - if (LOG.isDebugEnabled()) { - LOG.debug("Creating CuratorService with connection {}", + LOG.info("Creating CuratorService with connection {}", connectionDescription); - } + CuratorFramework framework; synchronized (CuratorService.class) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java index 271ab25..4c911da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java @@ -107,8 +107,10 @@ public class RegistryOperationsService extends CuratorService validatePath(path); // validate the record before putting it RegistryTypeUtils.validateServiceRecord(path, record); - LOG.info("Bound at {} : {}", path, record); + if (LOG.isDebugEnabled()) { + LOG.debug("Bound at {} : ServiceRecord = {}", path, record); + } CreateMode mode = CreateMode.PERSISTENT; byte[] bytes = serviceRecordMarshal.toBytes(record); zkSet(path, mode, bytes, getClientAcls(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 368832a..bcd1a9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; -import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.security.AccessRequest; @@ -65,7 +64,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -349,36 +347,6 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, createAndPopulateNewRMApp(appContext, appState.getSubmitTime(), appState.getUser(), true, appState.getStartTime()); - // If null amReq has been returned, check if it is the case that - // application has specified node label expression while node label - // has been disabled. Reject the recovery of this application if it - // is true and give clear message so that user can react properly. - if (!appContext.getUnmanagedAM() && - (application.getAMResourceRequests() == null || - application.getAMResourceRequests().isEmpty()) && - !YarnConfiguration.areNodeLabelsEnabled(this.conf)) { - // check application submission context and see if am resource request - // or application itself contains any node label expression. - List<ResourceRequest> amReqsFromAppContext = - appContext.getAMContainerResourceRequests(); - String labelExp = - (amReqsFromAppContext != null && !amReqsFromAppContext.isEmpty()) ? - amReqsFromAppContext.get(0).getNodeLabelExpression() : null; - if (labelExp == null) { - labelExp = appContext.getNodeLabelExpression(); - } - if (labelExp != null && - !labelExp.equals(RMNodeLabelsManager.NO_LABEL)) { - String message = "Failed to recover application " + appId - + ". NodeLabel is not enabled in cluster, but AM resource request " - + "contains a label expression."; - LOG.warn(message); - application.handle( - new RMAppEvent(appId, RMAppEventType.APP_REJECTED, message)); - return; - } - } - application.handle(new RMAppRecoverEvent(appId, rmState)); } @@ -398,28 +366,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, } ApplicationId applicationId = submissionContext.getApplicationId(); - List<ResourceRequest> amReqs = null; - try { - amReqs = validateAndCreateResourceRequest(submissionContext, isRecovery); - } catch (InvalidLabelResourceRequestException e) { - // This can happen if the application had been submitted and run - // with Node Label enabled but recover with Node Label disabled. - // Thus there might be node label expression in the application's - // resource requests. If this is the case, create RmAppImpl with - // null amReq and reject the application later with clear error - // message. So that the application can still be tracked by RM - // after recovery and user can see what's going on and react accordingly. - if (isRecovery && - !YarnConfiguration.areNodeLabelsEnabled(this.conf)) { - if (LOG.isDebugEnabled()) { - LOG.debug("AMResourceRequest is not created for " + applicationId - + ". NodeLabel is not enabled in cluster, but AM resource " - + "request contains a label expression."); - } - } else { - throw e; - } - } + List<ResourceRequest> amReqs = validateAndCreateResourceRequest( + submissionContext, isRecovery); // Verify and get the update application priority and set back to // submissionContext http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java index f10e885..e748955 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java @@ -18,15 +18,15 @@ package org.apache.hadoop.yarn.server.resourcemanager.blacklist; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; - /** * Maintains a list of failed nodes and returns that as long as number of * blacklisted nodes is below a threshold percentage of total nodes. If more http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 4210c54..c00f505 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -38,7 +38,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; import javax.crypto.SecretKey; -import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -110,6 +109,8 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition; import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; +import org.apache.hadoop.yarn.util.Apps; +import org.apache.hadoop.yarn.util.BoundedAppender; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; @@ -1552,38 +1553,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } } - private static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) { - switch (exitStatus) { - case ContainerExitStatus.PREEMPTED: - case ContainerExitStatus.KILLED_BY_RESOURCEMANAGER: - case ContainerExitStatus.KILLED_BY_APPMASTER: - case ContainerExitStatus.KILLED_AFTER_APP_COMPLETION: - case ContainerExitStatus.ABORTED: - // Neither the app's fault nor the system's fault. This happens by design, - // so no need for skipping nodes - return false; - case ContainerExitStatus.DISKS_FAILED: - // This container is marked with this exit-status means that the node is - // already marked as unhealthy given that most of the disks failed. So, no - // need for any explicit skipping of nodes. - return false; - case ContainerExitStatus.KILLED_EXCEEDED_VMEM: - case ContainerExitStatus.KILLED_EXCEEDED_PMEM: - // No point in skipping the node as it's not the system's fault - return false; - case ContainerExitStatus.SUCCESS: - return false; - case ContainerExitStatus.INVALID: - // Ideally, this shouldn't be considered for skipping a node. But in - // reality, it seems like there are cases where we are not setting - // exit-code correctly and so it's better to be conservative. See - // YARN-4284. - return true; - default: - return true; - } - } - private static final class UnmanagedAMAttemptSavedTransition extends AMLaunchedTransition { @Override @@ -1963,7 +1932,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { containerFinishedEvent.getContainerStatus(); if (containerStatus != null) { int exitStatus = containerStatus.getExitStatus(); - if (shouldCountTowardsNodeBlacklisting(exitStatus)) { + if (Apps.shouldCountTowardsNodeBlacklisting(exitStatus)) { appAttempt.addAMNodeToBlackList(nodeId); } } else { @@ -2279,114 +2248,4 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { return Collections.EMPTY_SET; } - /** - * A {@link CharSequence} appender that considers its {@link #limit} as upper - * bound. - * <p> - * When {@link #limit} would be reached on append, past messages will be - * truncated from head, and a header telling the user about truncation will be - * prepended, with ellipses in between header and messages. - * <p> - * Note that header and ellipses are not counted against {@link #limit}. - * <p> - * An example: - * - * <pre> - * {@code - * // At the beginning it's an empty string - * final Appendable shortAppender = new BoundedAppender(80); - * // The whole message fits into limit - * shortAppender.append( - * "message1 this is a very long message but fitting into limit\n"); - * // The first message is truncated, the second not - * shortAppender.append("message2 this is shorter than the previous one\n"); - * // The first message is deleted, the second truncated, the third - * // preserved - * shortAppender.append("message3 this is even shorter message, maybe.\n"); - * // The first two are deleted, the third one truncated, the last preserved - * shortAppender.append("message4 the shortest one, yet the greatest :)"); - * // Current contents are like this: - * // Diagnostic messages truncated, showing last 80 chars out of 199: - * // ...s is even shorter message, maybe. - * // message4 the shortest one, yet the greatest :) - * } - * </pre> - * <p> - * Note that <tt>null</tt> values are {@link #append(CharSequence) append}ed - * just like in {@link StringBuilder#append(CharSequence) original - * implementation}. - * <p> - * Note that this class is not thread safe. - */ - @VisibleForTesting - static class BoundedAppender { - @VisibleForTesting - static final String TRUNCATED_MESSAGES_TEMPLATE = - "Diagnostic messages truncated, showing last " - + "%d chars out of %d:%n...%s"; - - private final int limit; - private final StringBuilder messages = new StringBuilder(); - private int totalCharacterCount = 0; - - BoundedAppender(final int limit) { - Preconditions.checkArgument(limit > 0, "limit should be positive"); - - this.limit = limit; - } - - /** - * Append a {@link CharSequence} considering {@link #limit}, truncating - * from the head of {@code csq} or {@link #messages} when necessary. - * - * @param csq the {@link CharSequence} to append - * @return this - */ - BoundedAppender append(final CharSequence csq) { - appendAndCount(csq); - checkAndCut(); - - return this; - } - - private void appendAndCount(final CharSequence csq) { - final int before = messages.length(); - messages.append(csq); - final int after = messages.length(); - totalCharacterCount += after - before; - } - - private void checkAndCut() { - if (messages.length() > limit) { - final int newStart = messages.length() - limit; - messages.delete(0, newStart); - } - } - - /** - * Get current length of messages considering truncates - * without header and ellipses. - * - * @return current length - */ - int length() { - return messages.length(); - } - - /** - * Get a string representation of the actual contents, displaying also a - * header and ellipses when there was a truncate. - * - * @return String representation of the {@link #messages} - */ - @Override - public String toString() { - if (messages.length() < totalCharacterCount) { - return String.format(TRUNCATED_MESSAGES_TEMPLATE, messages.length(), - totalCharacterCount, messages.toString()); - } - - return messages.toString(); - } - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index c00b7be..b40ce40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstant import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; @@ -518,12 +519,20 @@ public abstract class AbstractYarnScheduler container.setVersion(status.getVersion()); ApplicationAttemptId attemptId = container.getId().getApplicationAttemptId(); + String labelExpression = status.getNodeLabelExpression(); + // If NodeLabel is disabled but recovered container has label expression + // its better to suppress that and considered as default label. + if (!status.getNodeLabelExpression().isEmpty() && !YarnConfiguration + .areNodeLabelsEnabled(rmContext.getYarnConfiguration())) { + labelExpression = RMNodeLabelsManager.NO_LABEL; + } + RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), attemptId, node.getNodeID(), applications.get( attemptId.getApplicationId()).getUser(), rmContext, - status.getCreationTime(), status.getNodeLabelExpression()); + status.getCreationTime(), labelExpression); return rmContainer; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index c67f1ce..7b554db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -22,6 +22,8 @@ import java.util.Set; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -55,7 +57,9 @@ import org.apache.hadoop.yarn.util.resource.Resources; @Private @Unstable public class SchedulerUtils { - + + private static final Log LOG = LogFactory.getLog(SchedulerUtils.class); + private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); @@ -200,9 +204,14 @@ public class SchedulerUtils { String labelExp = resReq.getNodeLabelExpression(); if (!(RMNodeLabelsManager.NO_LABEL.equals(labelExp) || null == labelExp)) { - throw new InvalidLabelResourceRequestException( - "Invalid resource request, node label not enabled " - + "but request contains label expression"); + String message = "NodeLabel is not enabled in cluster, but resource" + + " request contains a label expression."; + LOG.warn(message); + if (!isRecovery) { + throw new InvalidLabelResourceRequestException( + "Invalid resource request, node label not enabled " + + "but request contains label expression"); + } } } if (null == queueInfo) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 955b4b6..475c44e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -2589,14 +2589,14 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { } }; - // rm should successfully start with app1 loaded back in FAILED state - // due to node label not enabled but am resource request contains - // node label expression. + // rm should successfully start with app1 loaded back in SUCCESS state + // by pushing app to run default label for am container and let other + // containers to run normally. + try { rm2.start(); Assert.assertTrue("RM start successfully", true); Assert.assertEquals(1, rm2.getRMContext().getRMApps().size()); - rm2.waitForState(app1.getApplicationId(), RMAppState.FAILED); } catch (Exception e) { LOG.debug("Exception on start", e); Assert.fail("RM should start without any issue"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestBoundedAppender.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestBoundedAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestBoundedAppender.java deleted file mode 100644 index 9cb1e04..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestBoundedAppender.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import static org.junit.Assert.assertEquals; -import static org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl.BoundedAppender; - -/** - * Test class for {@link BoundedAppender}. - */ -public class TestBoundedAppender { - @Rule - public ExpectedException expected = ExpectedException.none(); - - @Test - public void initWithZeroLimitThrowsException() { - expected.expect(IllegalArgumentException.class); - expected.expectMessage("limit should be positive"); - - new BoundedAppender(0); - } - - @Test - public void nullAppendedNullStringRead() { - final BoundedAppender boundedAppender = new BoundedAppender(4); - boundedAppender.append(null); - - assertEquals("null appended, \"null\" read", "null", - boundedAppender.toString()); - } - - @Test - public void appendBelowLimitOnceValueIsReadCorrectly() { - final BoundedAppender boundedAppender = new BoundedAppender(2); - - boundedAppender.append("ab"); - - assertEquals("value appended is read correctly", "ab", - boundedAppender.toString()); - } - - @Test - public void appendValuesBelowLimitAreReadCorrectlyInFifoOrder() { - final BoundedAppender boundedAppender = new BoundedAppender(3); - - boundedAppender.append("ab"); - boundedAppender.append("cd"); - boundedAppender.append("e"); - boundedAppender.append("fg"); - - assertEquals("last values appended fitting limit are read correctly", - String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 7, "efg"), - boundedAppender.toString()); - } - - @Test - public void appendLastAboveLimitPreservesLastMessagePostfix() { - final BoundedAppender boundedAppender = new BoundedAppender(3); - - boundedAppender.append("ab"); - boundedAppender.append("cde"); - boundedAppender.append("fghij"); - - assertEquals( - "last value appended above limit postfix is read correctly", String - .format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 10, "hij"), - boundedAppender.toString()); - } - - @Test - public void appendMiddleAboveLimitPreservesLastMessageAndMiddlePostfix() { - final BoundedAppender boundedAppender = new BoundedAppender(3); - - boundedAppender.append("ab"); - boundedAppender.append("cde"); - - assertEquals("last value appended above limit postfix is read correctly", - String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 5, "cde"), - boundedAppender.toString()); - - boundedAppender.append("fg"); - - assertEquals( - "middle value appended above limit postfix and last value are " - + "read correctly", - String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 7, "efg"), - boundedAppender.toString()); - - boundedAppender.append("hijkl"); - - assertEquals( - "last value appended above limit postfix is read correctly", String - .format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 12, "jkl"), - boundedAppender.toString()); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java index 19b5dd9..295b59f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.util.BoundedAppender; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -86,7 +87,7 @@ public class TestRMAppAttemptImplDiagnostics { appAttempt.appendDiagnostics(beyondLimit); final String truncated = String.format( - RMAppAttemptImpl.BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 1024, + BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 1024, 1025, beyondLimit.substring(1)); assertEquals("messages beyond limit should be truncated", truncated, --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
