YARN-7410. Cleanup FixedValueResource to avoid dependency to ResourceUtils. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1700adc6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1700adc6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1700adc6 Branch: refs/heads/YARN-6592 Commit: 1700adc6f7e0ec9f855f554cd0d470a8f04e9c51 Parents: ad0fff2 Author: Sunil G <[email protected]> Authored: Fri Nov 3 07:25:29 2017 +0530 Committer: Sunil G <[email protected]> Committed: Fri Nov 3 07:25:29 2017 +0530 ---------------------------------------------------------------------- .../hadoop/yarn/api/records/Resource.java | 2 +- .../yarn/util/resource/ResourceUtils.java | 1 + .../hadoop/yarn/util/resource/Resources.java | 59 ++++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1700adc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 6bdde18..be292ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -364,7 +364,7 @@ public abstract class Resource implements Comparable<Resource> { } } - private void throwExceptionWhenArrayOutOfBound(int index) { + protected void throwExceptionWhenArrayOutOfBound(int index) { String exceptionMsg = String.format( "Trying to access ResourceInformation for given index=%d. " + "Acceptable index range is [0,%d), please check double check " http://git-wip-us.apache.org/repos/asf/hadoop/blob/1700adc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 8f75909..c9cc27b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -267,6 +267,7 @@ public class ResourceUtils { updateKnownResources(); updateResourceTypeIndex(); initializedResources = true; + numKnownResourceTypes = resourceTypes.size(); } private static void updateKnownResources() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/1700adc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 3690946..068e7f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -118,6 +118,65 @@ public class Resources { throw new RuntimeException(name + " cannot be modified!"); } + /* + * FixedValueResource cannot be updated when any resource types refresh + * by using approach introduced by YARN-7307 and do operations like + * Resources.compare(resource_x, Resources.none()) will throw exceptions. + * + * That's why we do reinitialize resource maps for following methods. + */ + + @Override + public ResourceInformation getResourceInformation(int index) + throws ResourceNotFoundException { + ResourceInformation ri = null; + try { + ri = super.getResourceInformation(index); + } catch (ResourceNotFoundException e) { + // Retry once to reinitialize resource information. + initResourceMap(); + try { + return super.getResourceInformation(index); + } catch (ResourceNotFoundException ee) { + throwExceptionWhenArrayOutOfBound(index); + } + } + return ri; + } + + @Override + public ResourceInformation getResourceInformation(String resource) + throws ResourceNotFoundException { + ResourceInformation ri; + try { + ri = super.getResourceInformation(resource); + } catch (ResourceNotFoundException e) { + // Retry once to reinitialize resource information. + initResourceMap(); + try { + return super.getResourceInformation(resource); + } catch (ResourceNotFoundException ee) { + throw ee; + } + } + return ri; + } + + @Override + public ResourceInformation[] getResources() { + if (resources.length != ResourceUtils.getNumberOfKnownResourceTypes()) { + // Retry once to reinitialize resource information. + initResourceMap(); + if (resources.length != ResourceUtils.getNumberOfKnownResourceTypes()) { + throw new ResourceNotFoundException("Failed to reinitialize " + + "FixedValueResource to get number of resource types same " + + "as configured"); + } + } + + return resources; + } + private void initResourceMap() { ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); if (types != null) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
