[hadoop] 02/02: YARN-10587. Fix AutoCreateLeafQueueCreation cap related caculation when in absolute mode. (Qi Zhu via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git commit 4f008153ef5fca9e1f71ebc7069c502e803ab1e8 Author: Wangda Tan AuthorDate: Thu Jan 21 09:31:14 2021 -0800 YARN-10587. Fix AutoCreateLeafQueueCreation cap related caculation when in absolute mode. (Qi Zhu via wangda) Change-Id: I8889c393a9f08b633c72a2e873c6f45ca12cc82b --- .../GuaranteedOrZeroCapacityOverTimePolicy.java | 9 ++--- .../capacity/TestAbsoluteResourceConfiguration.java | 12 +++- .../capacity/TestAbsoluteResourceWithAutoQueue.java | 12 ++-- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java index ab99317..a53fe4a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .queuemanagement; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .QueueManagementDynamicEditPolicy; @@ -624,7 +625,7 @@ public class GuaranteedOrZeroCapacityOverTimePolicy QueueCapacities capacities = leafQueueEntitlements.get( leafQueue.getQueuePath()); - updateToZeroCapacity(capacities, nodeLabel); + updateToZeroCapacity(capacities, nodeLabel, (LeafQueue)childQueue); deactivatedQueues.put(leafQueue.getQueuePath(), leafQueueTemplateCapacities); } @@ -822,7 +823,7 @@ public class GuaranteedOrZeroCapacityOverTimePolicy updateCapacityFromTemplate(capacities, nodeLabel); activate(leafQueue, nodeLabel); } else{ - updateToZeroCapacity(capacities, nodeLabel); + updateToZeroCapacity(capacities, nodeLabel, leafQueue); } } @@ -834,10 +835,12 @@ public class GuaranteedOrZeroCapacityOverTimePolicy } private void updateToZeroCapacity(QueueCapacities capacities, - String nodeLabel) { + String nodeLabel, LeafQueue leafQueue) { capacities.setCapacity(nodeLabel, 0.0f); capacities.setMaximumCapacity(nodeLabel, leafQueueTemplateCapacities.getMaximumCapacity(nodeLabel)); +leafQueue.getQueueResourceQuotas(). +setConfiguredMinResource(nodeLabel, Resource.newInstance(0, 0)); } private void updateCapacityFromTemplate(QueueCapacities capacities, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index da13e18..97f9a65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -231,8 +231,10 @@ public class TestAbsoluteResourceConfiguration { d1.queueResourceQuotas.getEffectiveMaxResource()); /** - * After adding child queue d2, adjustment happens and both d1 and d2 shares - * resource of Managed Parent Queue + * After adding child queue d2, d1 + d2 > resource + * of Managed Parent queue, d2 will change to 0. + * d1 will occupy all entire resource + * of Managed Parent queue. */ AutoCreatedLeafQueue d2 = new AutoCreatedLeafQueue(cs, "d2", parentQueue); cs.addQueue(d2); @@ -240,9 +242,9 @@
[hadoop] branch trunk updated (f1766e5 -> 4f00815)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from f1766e5 YARN-10596. Allow static definition of childless ParentQueues with auto-queue-creation-v2 enabled. Contributed by Andras Gyori new b7384a8 YARN-10531. Be able to disable user limit factor for CapacityScheduler Leaf Queue. (Qi Zhu via wangda) new 4f00815 YARN-10587. Fix AutoCreateLeafQueueCreation cap related caculation when in absolute mode. (Qi Zhu via wangda) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../scheduler/capacity/AbstractCSQueue.java| 9 +- .../scheduler/capacity/LeafQueue.java | 26 - .../scheduler/capacity/ParentQueue.java| 4 +- .../scheduler/capacity/PlanQueue.java | 3 + .../scheduler/capacity/UsersManager.java | 12 ++- .../GuaranteedOrZeroCapacityOverTimePolicy.java| 9 +- .../TestAbsoluteResourceConfiguration.java | 12 ++- .../TestAbsoluteResourceWithAutoQueue.java | 12 +-- .../TestCapacitySchedulerNewQueueAutoCreation.java | 35 +++ .../scheduler/capacity/TestLeafQueue.java | 110 - 10 files changed, 209 insertions(+), 23 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 01/02: YARN-10531. Be able to disable user limit factor for CapacityScheduler Leaf Queue. (Qi Zhu via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git commit b7384a8d02ae3cace8c7b5eae84f8f3916a0b177 Author: Wangda Tan AuthorDate: Thu Jan 21 09:27:37 2021 -0800 YARN-10531. Be able to disable user limit factor for CapacityScheduler Leaf Queue. (Qi Zhu via wangda) Change-Id: I670e5525619b320745254609c48e7e1afb084835 --- .../scheduler/capacity/AbstractCSQueue.java| 9 +- .../scheduler/capacity/LeafQueue.java | 26 - .../scheduler/capacity/ParentQueue.java| 4 +- .../scheduler/capacity/PlanQueue.java | 3 + .../scheduler/capacity/UsersManager.java | 12 ++- .../TestCapacitySchedulerNewQueueAutoCreation.java | 35 +++ .../scheduler/capacity/TestLeafQueue.java | 110 - 7 files changed, 190 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index bc3ff22..12ce05f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -1535,8 +1535,13 @@ public abstract class AbstractCSQueue implements CSQueue { leafQueue.setMaxApplications(maxApplications); int maxApplicationsPerUser = Math.min(maxApplications, - (int) (maxApplications * (leafQueue.getUsersManager().getUserLimit() - / 100.0f) * leafQueue.getUsersManager().getUserLimitFactor())); + (int) (maxApplications + * (leafQueue.getUsersManager().getUserLimit() / 100.0f) + * leafQueue.getUsersManager().getUserLimitFactor())); + if (leafQueue.getUsersManager().getUserLimitFactor() == -1) { +maxApplicationsPerUser = maxApplications; + } + leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser); LOG.info("LeafQueue:" + leafQueue.getQueuePath() + ", maxApplications=" + maxApplications + ", maxApplicationsPerUser=" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 15c321f..6bf8d0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -708,16 +708,33 @@ public class LeafQueue extends AbstractCSQueue { queueCapacities.getMaxAMResourcePercentage(nodePartition) * effectiveUserLimit * usersManager.getUserLimitFactor(), minimumAllocation); + + if (getUserLimitFactor() == -1) { +userAMLimit = Resources.multiplyAndNormalizeUp( +resourceCalculator, queuePartitionResource, +queueCapacities.getMaxAMResourcePercentage(nodePartition), +minimumAllocation); + } + userAMLimit = Resources.min(resourceCalculator, lastClusterResource, userAMLimit, Resources.clone(getAMResourceLimitPerPartition(nodePartition))); - Resource preWeighteduserAMLimit = Resources.multiplyAndNormalizeUp( + Resource preWeighteduserAMLimit = + Resources.multiplyAndNormalizeUp( resourceCalculator, queuePartitionResource, queueCapacities.getMaxAMResourcePercentage(nodePartition) * preWeightedUserLimit * usersManager.getUserLimitFactor(), minimumAllocation); + + if (getUserLimitFactor() == -1) { +preWeighteduserAMLimit = Resources.multiplyAndNormalizeUp( +resourceCalculator, queuePartitionResource, +queueCapacities.getMaxAMResourcePercentage(nodePartition), +minimumAllocation); + } + preWeighteduserAMLimit = Resources.min(resourceCalculator, lastClusterResource, preWeighteduserAMLimit, @@ -1896,9 +191
[hadoop] branch trunk updated: YARN-10506. Update queue creation logic to use weight mode and allow the flexible static/dynamic creation. (Contributed by Qi Zhu, Andras Gyori)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 3d46141 YARN-10506. Update queue creation logic to use weight mode and allow the flexible static/dynamic creation. (Contributed by Qi Zhu, Andras Gyori) 3d46141 is described below commit 3d46141583e357ee12181fc9084ef6234bedf3f3 Author: Wangda Tan AuthorDate: Fri Jan 15 14:20:08 2021 -0800 YARN-10506. Update queue creation logic to use weight mode and allow the flexible static/dynamic creation. (Contributed by Qi Zhu, Andras Gyori) Change-Id: I118862fd5e11ee6888275e2bcf667fedfa56c5d7 --- .../placement/ApplicationPlacementContext.java | 13 + .../scheduler/capacity/AbstractCSQueue.java| 54 ++- .../capacity/AbstractManagedParentQueue.java | 2 +- .../scheduler/capacity/CSQueueUtils.java | 18 +- .../scheduler/capacity/CapacityScheduler.java | 84 ++-- .../CapacitySchedulerAutoQueueHandler.java | 127 ++ .../capacity/CapacitySchedulerConfigValidator.java | 14 +- .../capacity/CapacitySchedulerConfiguration.java | 27 +- .../capacity/CapacitySchedulerQueueManager.java| 35 +- .../scheduler/capacity/LeafQueue.java | 17 +- .../scheduler/capacity/ParentQueue.java| 165 +++- .../scheduler/capacity/PlanQueue.java | 3 +- .../scheduler/capacity/QueueCapacities.java| 2 +- .../TestCapacitySchedulerAutoQueueCreation.java| 25 +- .../TestCapacitySchedulerNewQueueAutoCreation.java | 436 + .../scheduler/capacity/TestLeafQueue.java | 4 + 16 files changed, 927 insertions(+), 99 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java index f2f92b8..3ae9ac4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java @@ -42,6 +42,10 @@ public class ApplicationPlacementContext { return queue; } + public void setQueue(String q) { +queue = q; + } + public String getParentQueue() { return parentQueue; } @@ -49,4 +53,13 @@ public class ApplicationPlacementContext { public boolean hasParentQueue() { return parentQueue != null; } + + public String getFullQueuePath() { +if (parentQueue != null) { + return parentQueue + "." + queue; +} else { + return queue; +} + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 9e7b0d8..fd144f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -151,6 +151,14 @@ public abstract class AbstractCSQueue implements CSQueue { private Map userWeights = new HashMap(); private int maxParallelApps; + // is it a dynamic queue? + private boolean dynamicQueue = false; + + // When this queue has application submit to? + // This property only applies to dynamic queue, + // and will be used to check when the queue need to be removed. + private long lastSubmittedTimestamp; + public AbstractCSQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { this(cs, cs.getConfiguration(), queueName, parent, old); @@ -172,7 +180,7 @@ public abstract class AbstractCSQueue implements CSQueue { this.metrics = old != null ? (CSQueueMetrics) old.getMetrics() : CSQueueMetrics.forQueue(getQueuePath(), parent, -cs.getConfiguration().getEnableUserMetrics(), cs.getConf()); +cs.getConfiguration().getEnableUserMetrics(), configuration); this.csCo
[hadoop] branch trunk updated: YARN-10504. Implement weight mode in Capacity Scheduler. (Contributed by Wangda Tan, Benjamin Teke, zhuqi, Andras Gyori)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new b0eec09 YARN-10504. Implement weight mode in Capacity Scheduler. (Contributed by Wangda Tan, Benjamin Teke, zhuqi, Andras Gyori) b0eec09 is described below commit b0eec0909772cf92427957670da5630b1dd11da0 Author: Wangda Tan AuthorDate: Mon Jan 11 17:46:09 2021 -0800 YARN-10504. Implement weight mode in Capacity Scheduler. (Contributed by Wangda Tan, Benjamin Teke, zhuqi, Andras Gyori) Change-Id: Ic49c730b0ab502ba86527fb662d25c4c8b1c2588 --- .../capacity/AbstractAutoCreatedLeafQueue.java | 5 - .../scheduler/capacity/AbstractCSQueue.java| 242 +++-- .../scheduler/capacity/AutoCreatedLeafQueue.java | 16 - .../scheduler/capacity/CSQueueUtils.java | 124 ++--- .../capacity/CapacitySchedulerConfiguration.java | 88 +++- .../capacity/CapacitySchedulerQueueManager.java| 4 + .../scheduler/capacity/LeafQueue.java | 50 +- .../scheduler/capacity/ManagedParentQueue.java | 20 +- .../scheduler/capacity/ParentQueue.java| 541 ++--- .../scheduler/capacity/QueueCapacities.java| 54 +- .../scheduler/capacity/ReservationQueue.java | 4 +- .../GuaranteedOrZeroCapacityOverTimePolicy.java| 29 ++ .../TestAbsoluteResourceConfiguration.java | 7 +- .../TestAbsoluteResourceWithAutoQueue.java | 10 +- .../capacity/TestCSMaxRunningAppsEnforcer.java | 12 +- .../TestCapacitySchedulerAutoCreatedQueueBase.java | 12 +- .../capacity/TestCapacitySchedulerWeightMode.java | 452 + .../scheduler/capacity/TestLeafQueue.java | 11 +- .../scheduler/capacity/TestParentQueue.java| 9 +- .../scheduler/capacity/TestQueueCapacities.java| 21 +- .../scheduler/capacity/TestQueueParsing.java | 55 ++- .../scheduler/capacity/TestReservationQueue.java | 18 +- .../resourcemanager/webapp/TestRMWebServices.java | 2 +- .../TestRMWebServicesForCSWithPartitions.java | 16 +- 24 files changed, 1301 insertions(+), 501 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java index 2b22241..8d77334 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java @@ -128,9 +128,4 @@ public class AbstractAutoCreatedLeafQueue extends LeafQueue { writeLock.unlock(); } } - - protected void setupConfigurableCapacities(QueueCapacities queueCapacities) { -CSQueueUtils.updateAndCheckCapacitiesByLabel(getQueuePath(), -queueCapacities, parent == null ? null : parent.getQueueCapacities()); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 8d22a36..9e7b0d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -100,7 +100,7 @@ public abstract class AbstractCSQueue implements CSQueue { String defaultLabelExpression; private String multiNodeSortingPolicyName = null; - Map acls = + Map acls = new HashMap(); volatile boolean reservationsContinueLooking; private volatile boolean preemptionDisabled; @@ -112,7 +112,7 @@ public abstract class AbstractCSQueue implements CSQueue { volatile ResourceUsage queueUsage; private final boolean fullPathQueueNamingPolicy = false; - + // Track capacities like used-capcity/abs-used-capacity/capacity/abs-capacity, // etc. QueueCapacities queueCapacities; @@ -129,12 +129,15 @@ public abstract class AbstractCSQueue
[hadoop] branch trunk updated: YARN-10458. Hive On Tez queries fails upon submission to dynamically created pools. (Peter Bacsko via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new c47c9fd YARN-10458. Hive On Tez queries fails upon submission to dynamically created pools. (Peter Bacsko via wangda) c47c9fd is described below commit c47c9fd65d835fc53c9eef59a05b97335b04e320 Author: Wangda Tan AuthorDate: Fri Oct 30 09:49:48 2020 -0700 YARN-10458. Hive On Tez queries fails upon submission to dynamically created pools. (Peter Bacsko via wangda) Change-Id: I518dc925187ce55e9d35a37ba20878c0f4e37e5c --- .../yarn/server/resourcemanager/RMAppManager.java | 4 + .../scheduler/capacity/CapacityScheduler.java | 20 + .../server/resourcemanager/TestAppManager.java | 94 +- .../TestCapacitySchedulerAutoQueueCreation.java| 64 ++- .../capacity/TestCapacitySchedulerQueueACLs.java | 42 ++ 5 files changed, 219 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 1e62f25..fe18d82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -927,6 +927,10 @@ public class RMAppManager implements EventHandler, return usernameUsedForPlacement; } String queue = appPlacementContext.getQueue(); + String parent = appPlacementContext.getParentQueue(); + if (scheduler instanceof CapacityScheduler && parent != null) { +queue = parent + "." + queue; + } if (callerUGI != null && scheduler .checkAccess(callerUGI, QueueACL.SUBMIT_APPLICATIONS, queue)) { usernameUsedForPlacement = userNameFromAppTag; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index a632bfa..259cd5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2290,6 +2290,21 @@ public class CapacityScheduler extends public boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, String queueName) { CSQueue queue = getQueue(queueName); + +if (queueName.startsWith("root.")) { + // can only check proper ACLs if the path is fully qualified + while (queue == null) { +int sepIndex = queueName.lastIndexOf("."); +String parentName = queueName.substring(0, sepIndex); +if (LOG.isDebugEnabled()) { + LOG.debug("Queue {} does not exist, checking parent {}", + queueName, parentName); +} +queueName = parentName; +queue = queueManager.getQueue(queueName); + } +} + if (queue == null) { LOG.debug("ACL not found for queue access-type {} for queue {}", acl, queueName); @@ -3307,4 +3322,9 @@ public class CapacityScheduler extends public void setActivitiesManagerEnabled(boolean enabled) { this.activitiesManagerEnabled = enabled; } + + @VisibleForTesting + public void setQueueManager(CapacitySchedulerQueueManager qm) { +this.queueManager = qm; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index bc89917..e8b4105 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-
[hadoop] branch branch-3.1 updated: YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam Antal/Eric Yang via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.1 by this push: new 0ee7d09 YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam Antal/Eric Yang via wangda) 0ee7d09 is described below commit 0ee7d091387ac458b0744e8087d5dad95fd5855f Author: Wangda Tan AuthorDate: Fri Sep 6 19:18:13 2019 -0700 YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam Antal/Eric Yang via wangda) Change-Id: I18b8edc930b2efa0652f59c246931ad0d46827f3 (cherry picked from commit 34b82e6da0a471010cdae613ba39487889d79369) --- .../org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index f54bd45..e3544ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -1178,8 +1178,7 @@ public class ResourceManager extends CompositeService if (null == onDiskPath) { String war = "hadoop-yarn-ui-" + VersionInfo.getVersion() + ".war"; -URLClassLoader cl = (URLClassLoader) ClassLoader.getSystemClassLoader(); -URL url = cl.findResource(war); +URL url = getClass().getClassLoader().getResource(war); if (null == url) { onDiskPath = getWebAppsPath("ui2"); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam Antal/Eric Yang via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 0e77347 YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam Antal/Eric Yang via wangda) 0e77347 is described below commit 0e77347972da9193b35b07a861b3d15d58c0ab47 Author: Wangda Tan AuthorDate: Fri Sep 6 19:18:13 2019 -0700 YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam Antal/Eric Yang via wangda) Change-Id: I18b8edc930b2efa0652f59c246931ad0d46827f3 (cherry picked from commit 34b82e6da0a471010cdae613ba39487889d79369) --- .../org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 41edfe4..65b0229 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -1202,8 +1202,7 @@ public class ResourceManager extends CompositeService if (null == onDiskPath) { String war = "hadoop-yarn-ui-" + VersionInfo.getVersion() + ".war"; -URLClassLoader cl = (URLClassLoader) ClassLoader.getSystemClassLoader(); -URL url = cl.findResource(war); +URL url = getClass().getClassLoader().getResource(war); if (null == url) { onDiskPath = getWebAppsPath("ui2"); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam Antal/Eric Yang via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 34b82e6 YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam Antal/Eric Yang via wangda) 34b82e6 is described below commit 34b82e6da0a471010cdae613ba39487889d79369 Author: Wangda Tan AuthorDate: Fri Sep 6 19:18:13 2019 -0700 YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam Antal/Eric Yang via wangda) Change-Id: I18b8edc930b2efa0652f59c246931ad0d46827f3 --- .../org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 57be468..d939ac1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -1232,8 +1232,7 @@ public class ResourceManager extends CompositeService if (null == onDiskPath) { String war = "hadoop-yarn-ui-" + VersionInfo.getVersion() + ".war"; -URLClassLoader cl = (URLClassLoader) ClassLoader.getSystemClassLoader(); -URL url = cl.findResource(war); +URL url = getClass().getClassLoader().getResource(war); if (null == url) { onDiskPath = getWebAppsPath("ui2"); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-9319. Fix compilation issue of handling typedef an existing name by gcc compiler. (Zhankun Tang via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new b17a260 YARN-9319. Fix compilation issue of handling typedef an existing name by gcc compiler. (Zhankun Tang via wangda) b17a260 is described below commit b17a2602d1fcece0979feb1438a3e71099126bef Author: Wangda Tan AuthorDate: Thu Feb 21 11:17:32 2019 -0800 YARN-9319. Fix compilation issue of handling typedef an existing name by gcc compiler. (Zhankun Tang via wangda) Change-Id: Ie8293cc5b710ff355fb6a1ea7201be53ed0385bf --- .../native/container-executor/impl/modules/devices/devices-module.c | 4 ++-- .../native/container-executor/impl/modules/devices/devices-module.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c index 9df6662..4063bae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c @@ -82,7 +82,7 @@ cleanup: } static int internal_handle_devices_request( -update_cgroups_parameters_function update_cgroups_parameters_func_p, +update_cgroups_param_function update_cgroups_parameters_func_p, char** deny_devices_number_tokens, char** allow_devices_number_tokens, const char* container_id) { @@ -196,7 +196,7 @@ void reload_devices_configuration() { * --allowed_devices 8:32,8:48,243:2 \ * --container_id container_x_y */ -int handle_devices_request(update_cgroups_parameters_function func, +int handle_devices_request(update_cgroups_param_function func, const char* module_name, int module_argc, char** module_argv) { if (!cfg_section) { reload_devices_configuration(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h index c5d6785..552d070 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h @@ -28,13 +28,13 @@ #define DEVICES_MODULE_SECTION_NAME "devices" // For unit test stubbing -typedef int (*update_cgroups_parameters_function)(const char*, const char*, +typedef int (*update_cgroups_param_function)(const char*, const char*, const char*, const char*); /** * Handle devices requests */ -int handle_devices_request(update_cgroups_parameters_function func, +int handle_devices_request(update_cgroups_param_function func, const char* module_name, int module_argc, char** module_argv); /** - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] annotated tag rel/release-3.1.2 created (now b156fd9)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to annotated tag rel/release-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git. at b156fd9 (tag) tagging 1019dde65bcf12e05ef48ac71e84550d589e5d9a (commit) replaces release-3.1.2-RC0 by Wangda Tan on Tue Feb 5 13:26:33 2019 -0800 - Log - Hadoop 3.1.2 release -BEGIN PGP SIGNATURE- iQJGBAABCAAwFiEETImYU83aTkDGAhK1s/plPVcwDUUFAlxZ/4kSHHdhbmdkYUBh cGFjaGUub3JnAAoJELP6ZT1XMA1FqG8P/0/Mp2poBQEWtJVbL7VoQ3P2OMTyBvcy iyvrbjwa+QlxB/mFJLq9S05jPr8htwcukNi8AtbZgtxs1ovuBhtVu9bzbqbpSxlm aVAWbQ7pqu78/wSRKyTcS7JTikxZ2F4E9IEH18kZJcaGZzUHqDwq04phj5157BWk 5Xt7ilrM8W+4csA9K0BM1LUJPRfu9TzW/5HikOa0i/JWj/0NNzN9vvYjB0nLWRPj sp3d2FvDLQuJlkLnnLXIoGj9MIP4Qy2c3VhKjLeWHdIE6vMXG+xQqS/Kd5FFCIpy HtzdqeTk04WUCQ+ym0YBcmOtNcUn4yrR5fLab6rmyDBnMS7qZZ8dNp4KUBv8KBjt eiA5YgBfBDai0N4UXQCvwOD9g+KXmHsRXePrcvndCNB1k6g4oGM9uCHFxoKFu6/G eeaUR0KBOkGswD4hKX+BrvQnePs00IaI1m5+8LexIjY7HPuDbGBG0Co6y8mndJQq gvCNiCRV7FZlOWlxM+bUw4+G2lbOQXh5zrm2fK9boy51MuDF2zwpDD7mWFeldn8a kyAJx1W/P9LaEGgkDc2P19xIwxFsSWsoOt6RlrSxJeJsbWveAql6IBWCRGtgRk8R Vnkf9yoMVCmrMO1g/znkgHVafMP3xUU9T+xYMzKjKNW+IYA4moVjt9LtcnZA+4Ql i4UfrqzD0Jcn =Di3L -END PGP SIGNATURE- --- No new revisions were added by this update. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 02/04: YARN-8747. [UI2] YARN UI2 page loading failed due to js error under some time zone configuration. Contributed by collinma.
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit db87c107abbad9a3c6e34c2927e60f0d01a56079 Author: Akira Ajisaka AuthorDate: Wed Jan 16 14:35:07 2019 +0900 YARN-8747. [UI2] YARN UI2 page loading failed due to js error under some time zone configuration. Contributed by collinma. (cherry picked from commit 104ef5df36fd45f1c2d07a2f3d441263aa85e22e) (cherry picked from commit d07d275f942dcd9f65a8c867ffedd024c9307e0c) --- .../hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json index b7591a5..8fa9668 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json @@ -13,7 +13,7 @@ "qunit": "1.19.0", "jquery-ui": "1.11.4", "moment": "2.12.0", -"moment-timezone": "0.5.0", +"moment-timezone": "0.5.1", "more-js": "0.8.2", "bootstrap": "3.3.6", "d3": "~3.5.6", - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 01/04: YARN-9173. FairShare calculation broken for large values after YARN-8833. Contributed Wilfred Spiegelenburg.
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit f6617d4c8bd8f5d226169e6870e703049eb456df Author: Weiwei Yang AuthorDate: Tue Jan 8 13:56:21 2019 +0800 YARN-9173. FairShare calculation broken for large values after YARN-8833. Contributed Wilfred Spiegelenburg. (cherry picked from commit 91e9c9f96eca01bbef7d22b4cff1ced3be1ac5e8) --- .../scheduler/fair/policies/ComputeFairShares.java | 62 .../scheduler/fair/FakeSchedulable.java| 13 ++- .../scheduler/fair/TestComputeFairShares.java | 104 ++--- 3 files changed, 125 insertions(+), 54 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java index 3fe0c68..97bb4c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java @@ -24,6 +24,8 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable; +import static java.lang.Math.addExact; + /** * Contains logic for computing the fair shares. A {@link Schedulable}'s fair * share is {@link Resource} it is entitled to, independent of the current @@ -31,10 +33,13 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable; * consumption lies at or below its fair share will never have its containers * preempted. */ -public class ComputeFairShares { +public final class ComputeFairShares { private static final int COMPUTE_FAIR_SHARES_ITERATIONS = 25; + private ComputeFairShares() { + } + /** * Compute fair share of the given schedulables.Fair share is an allocation of * shares considering only active schedulables ie schedulables which have @@ -100,19 +105,20 @@ public class ComputeFairShares { * all Schedulables are only given their minShare) and an upper bound computed * to be large enough that too many slots are given (by doubling R until we * use more than totalResources resources). The helper method - * resourceUsedWithWeightToResourceRatio computes the total resources used with a - * given value of R. + * resourceUsedWithWeightToResourceRatio computes the total resources used + * with a given value of R. * * The running time of this algorithm is linear in the number of Schedulables, - * because resourceUsedWithWeightToResourceRatio is linear-time and the number of - * iterations of binary search is a constant (dependent on desired precision). + * because resourceUsedWithWeightToResourceRatio is linear-time and the + * number of iterations of binary search is a constant (dependent on desired + * precision). */ private static void computeSharesInternal( Collection allSchedulables, Resource totalResources, String type, boolean isSteadyShare) { Collection schedulables = new ArrayList<>(); -int takenResources = handleFixedFairShares( +long takenResources = handleFixedFairShares( allSchedulables, schedulables, isSteadyShare, type); if (schedulables.isEmpty()) { @@ -121,12 +127,11 @@ public class ComputeFairShares { // Find an upper bound on R that we can use in our binary search. We start // at R = 1 and double it until we have either used all the resources or we // have met all Schedulables' max shares. -int totalMaxShare = 0; +long totalMaxShare = 0; for (Schedulable sched : schedulables) { long maxShare = sched.getMaxShare().getResourceValue(type); - totalMaxShare = (int) Math.min(maxShare + (long)totalMaxShare, - Integer.MAX_VALUE); - if (totalMaxShare == Integer.MAX_VALUE) { + totalMaxShare = safeAdd(maxShare, totalMaxShare); + if (totalMaxShare == Long.MAX_VALUE) { break; } } @@ -166,23 +171,24 @@ public class ComputeFairShares { target = sched.getFairShare(); } - target.setResourceValue(type, (long)computeShare(sched, right, type)); + target.setResourceValue(type, computeShare(sched, right, type)); } } /** * Compute the resources that would be used given a weight-t
[hadoop] branch branch-3.1.2 updated (9f88af7 -> 211e2ca)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to branch branch-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 9f88af7 Revert "HDFS-14084. Need for more stats in DFSClient. Contributed by Pranay Singh." new f6617d4 YARN-9173. FairShare calculation broken for large values after YARN-8833. Contributed Wilfred Spiegelenburg. new db87c10 YARN-8747. [UI2] YARN UI2 page loading failed due to js error under some time zone configuration. Contributed by collinma. new dc4ca27 YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda) new 211e2ca YARN-9204. RM fails to start if absolute resource is specified for partition capacity in CS queues. Contributed by Jiandan Yang. The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../rmapp/attempt/RMAppAttemptImpl.java| 14 ++- .../capacity/CapacitySchedulerConfiguration.java | 24 - .../scheduler/fair/policies/ComputeFairShares.java | 62 .../rmapp/attempt/TestRMAppAttemptTransitions.java | 80 +++- .../scheduler/capacity/TestCapacityScheduler.java | 41 .../scheduler/fair/FakeSchedulable.java| 13 ++- .../scheduler/fair/TestComputeFairShares.java | 104 ++--- .../hadoop-yarn-ui/src/main/webapp/bower.json | 2 +- 8 files changed, 278 insertions(+), 62 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 04/04: YARN-9204. RM fails to start if absolute resource is specified for partition capacity in CS queues. Contributed by Jiandan Yang.
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit 211e2caef21fc386b798674fd0b7347d66fce837 Author: Weiwei Yang AuthorDate: Mon Jan 21 16:54:14 2019 +0800 YARN-9204. RM fails to start if absolute resource is specified for partition capacity in CS queues. Contributed by Jiandan Yang. (cherry picked from commit abde1e1f58d5b699e4b8e460cff68e154738169b) (cherry picked from commit 4edd883d4818dee3bcf2f8dea3e3d8ebf35d6605) --- .../capacity/CapacitySchedulerConfiguration.java | 24 +++-- .../scheduler/capacity/TestCapacityScheduler.java | 41 ++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 547d131..1a300d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -471,6 +471,17 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur ", capacity=" + capacity); } + @VisibleForTesting + public void setCapacity(String queue, String absoluteResourceCapacity) { +if (queue.equals("root")) { + throw new IllegalArgumentException( + "Cannot set capacity, root queue has a fixed capacity"); +} +set(getQueuePrefix(queue) + CAPACITY, absoluteResourceCapacity); +LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) ++ ", capacity=" + absoluteResourceCapacity); + } + public float getNonLabeledQueueMaximumCapacity(String queue) { String configuredCapacity = get(getQueuePrefix(queue) + MAXIMUM_CAPACITY); boolean matcher = (configuredCapacity != null) @@ -505,7 +516,13 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public void setCapacityByLabel(String queue, String label, float capacity) { setFloat(getNodeLabelPrefix(queue, label) + CAPACITY, capacity); } - + + @VisibleForTesting + public void setCapacityByLabel(String queue, String label, + String absoluteResourceCapacity) { +set(getNodeLabelPrefix(queue, label) + CAPACITY, absoluteResourceCapacity); + } + public void setMaximumCapacityByLabel(String queue, String label, float capacity) { setFloat(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY, capacity); @@ -642,8 +659,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur private float internalGetLabeledQueueCapacity(String queue, String label, String suffix, float defaultValue) { String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix; -boolean matcher = (capacityPropertyName != null) -&& RESOURCE_PATTERN.matcher(capacityPropertyName).find(); +String configuredCapacity = get(capacityPropertyName); +boolean matcher = (configuredCapacity != null) +&& RESOURCE_PATTERN.matcher(configuredCapacity).find(); if (matcher) { // Return capacity in percentage as 0 for non-root queues and 100 for // root.From AbstractCSQueue, absolute resource will be parsed and diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index db8836a..772e72e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -40,6 +40,7 @@ import java.util.Map; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; +import com.google.common.collect.Sets; import org.apache.
[hadoop] 03/04: YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit dc4ca276771ff0381780f944e3539b27d7499d4f Author: Wangda Tan AuthorDate: Thu Jan 17 14:20:10 2019 -0800 YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda) Change-Id: I4359f59a73a278a941f4bb9d106dd38c9cb471fe (cherry picked from commit 6d7eedfd28cc1712690db2f6ca8a281b0901ee28) (cherry picked from commit fe7cb2d84ac160c5fed00640d85e2c5c4c6d2412) (cherry picked from commit a685ffe9a9d5cf073faa2625d1125839fb57bae2) --- .../rmapp/attempt/RMAppAttemptImpl.java| 14 +++- .../rmapp/attempt/TestRMAppAttemptTransitions.java | 80 +- 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index bb43693..feb690f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -437,9 +437,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { RMAppAttemptState.FAILED, EnumSet.of( RMAppAttemptEventType.LAUNCHED, + RMAppAttemptEventType.LAUNCH_FAILED, RMAppAttemptEventType.EXPIRE, RMAppAttemptEventType.KILL, RMAppAttemptEventType.FAIL, + RMAppAttemptEventType.REGISTERED, RMAppAttemptEventType.UNREGISTERED, RMAppAttemptEventType.STATUS_UPDATE, RMAppAttemptEventType.CONTAINER_ALLOCATED)) @@ -1203,10 +1205,16 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } // Set the masterContainer - appAttempt.setMasterContainer(amContainerAllocation.getContainers() - .get(0)); + Container amContainer = amContainerAllocation.getContainers().get(0); RMContainerImpl rmMasterContainer = (RMContainerImpl)appAttempt.scheduler - .getRMContainer(appAttempt.getMasterContainer().getId()); + .getRMContainer(amContainer.getId()); + //while one NM is removed, the scheduler will clean the container,the + //following CONTAINER_FINISHED event will handle the cleaned container. + //so just return RMAppAttemptState.SCHEDULED + if (rmMasterContainer == null) { +return RMAppAttemptState.SCHEDULED; + } + appAttempt.setMasterContainer(amContainer); rmMasterContainer.setAMContainer(true); // The node set in NMTokenSecrentManager is used for marking whether the // NMToken has been issued for this node to the AM. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 4a5c671..faecdb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -986,7 +986,7 @@ public class TestRMAppAttemptTransitions { public void testAttemptAddedAtFinalSaving() { submitApplicationAttempt(); -// SUBNITED->FINAL_SAVING +// SUBMITTED->FINAL_SAVING applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt .getAppAttemptId(), RMAppAttemptEventType.KILL)); assertEquals(RMAppAttemptState.FINAL_SAVING, @@ -999,6 +999,56 @@ public class TestRMAppAttemptTransitions { applicationAttempt.getAppAttemptState()); } + @Test(timeout = 1) + public void testAttemptRegisteredAtFailed() { +Container amContainer = allocateApplicationAttempt(); +launchApplicationAttempt(amContainer); + +//send CONTAINER_FINISHED event +NodeId anyNodeId = NodeId.newInstance("
[hadoop] branch branch-3.1 updated: YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.1 by this push: new a685ffe YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda) a685ffe is described below commit a685ffe9a9d5cf073faa2625d1125839fb57bae2 Author: Wangda Tan AuthorDate: Thu Jan 17 14:20:10 2019 -0800 YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda) Change-Id: I4359f59a73a278a941f4bb9d106dd38c9cb471fe (cherry picked from commit 6d7eedfd28cc1712690db2f6ca8a281b0901ee28) (cherry picked from commit fe7cb2d84ac160c5fed00640d85e2c5c4c6d2412) --- .../rmapp/attempt/RMAppAttemptImpl.java| 14 +++- .../rmapp/attempt/TestRMAppAttemptTransitions.java | 80 +- 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index bb43693..feb690f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -437,9 +437,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { RMAppAttemptState.FAILED, EnumSet.of( RMAppAttemptEventType.LAUNCHED, + RMAppAttemptEventType.LAUNCH_FAILED, RMAppAttemptEventType.EXPIRE, RMAppAttemptEventType.KILL, RMAppAttemptEventType.FAIL, + RMAppAttemptEventType.REGISTERED, RMAppAttemptEventType.UNREGISTERED, RMAppAttemptEventType.STATUS_UPDATE, RMAppAttemptEventType.CONTAINER_ALLOCATED)) @@ -1203,10 +1205,16 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } // Set the masterContainer - appAttempt.setMasterContainer(amContainerAllocation.getContainers() - .get(0)); + Container amContainer = amContainerAllocation.getContainers().get(0); RMContainerImpl rmMasterContainer = (RMContainerImpl)appAttempt.scheduler - .getRMContainer(appAttempt.getMasterContainer().getId()); + .getRMContainer(amContainer.getId()); + //while one NM is removed, the scheduler will clean the container,the + //following CONTAINER_FINISHED event will handle the cleaned container. + //so just return RMAppAttemptState.SCHEDULED + if (rmMasterContainer == null) { +return RMAppAttemptState.SCHEDULED; + } + appAttempt.setMasterContainer(amContainer); rmMasterContainer.setAMContainer(true); // The node set in NMTokenSecrentManager is used for marking whether the // NMToken has been issued for this node to the AM. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 4a5c671..faecdb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -986,7 +986,7 @@ public class TestRMAppAttemptTransitions { public void testAttemptAddedAtFinalSaving() { submitApplicationAttempt(); -// SUBNITED->FINAL_SAVING +// SUBMITTED->FINAL_SAVING applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt .getAppAttemptId(), RMAppAttemptEventType.KILL)); assertEquals(RMAppAttemptState.FINAL_SAVING, @@ -999,6 +999,56 @@ public class TestRMAppAttemptTransitions { applicationAttempt.getAppAttemptState()); } + @Test(timeout = 1) + public void testAttemptRegisteredAt
[hadoop] branch branch-3.2 updated: YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new fe7cb2d YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda) fe7cb2d is described below commit fe7cb2d84ac160c5fed00640d85e2c5c4c6d2412 Author: Wangda Tan AuthorDate: Thu Jan 17 14:20:10 2019 -0800 YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda) Change-Id: I4359f59a73a278a941f4bb9d106dd38c9cb471fe (cherry picked from commit 6d7eedfd28cc1712690db2f6ca8a281b0901ee28) --- .../rmapp/attempt/RMAppAttemptImpl.java| 14 +++- .../rmapp/attempt/TestRMAppAttemptTransitions.java | 80 +- 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 3ec9c49..03039da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -437,9 +437,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { RMAppAttemptState.FAILED, EnumSet.of( RMAppAttemptEventType.LAUNCHED, + RMAppAttemptEventType.LAUNCH_FAILED, RMAppAttemptEventType.EXPIRE, RMAppAttemptEventType.KILL, RMAppAttemptEventType.FAIL, + RMAppAttemptEventType.REGISTERED, RMAppAttemptEventType.UNREGISTERED, RMAppAttemptEventType.STATUS_UPDATE, RMAppAttemptEventType.CONTAINER_ALLOCATED)) @@ -1203,10 +1205,16 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } // Set the masterContainer - appAttempt.setMasterContainer(amContainerAllocation.getContainers() - .get(0)); + Container amContainer = amContainerAllocation.getContainers().get(0); RMContainerImpl rmMasterContainer = (RMContainerImpl)appAttempt.scheduler - .getRMContainer(appAttempt.getMasterContainer().getId()); + .getRMContainer(amContainer.getId()); + //while one NM is removed, the scheduler will clean the container,the + //following CONTAINER_FINISHED event will handle the cleaned container. + //so just return RMAppAttemptState.SCHEDULED + if (rmMasterContainer == null) { +return RMAppAttemptState.SCHEDULED; + } + appAttempt.setMasterContainer(amContainer); rmMasterContainer.setAMContainer(true); // The node set in NMTokenSecrentManager is used for marking whether the // NMToken has been issued for this node to the AM. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 4a5c671..faecdb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -986,7 +986,7 @@ public class TestRMAppAttemptTransitions { public void testAttemptAddedAtFinalSaving() { submitApplicationAttempt(); -// SUBNITED->FINAL_SAVING +// SUBMITTED->FINAL_SAVING applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt .getAppAttemptId(), RMAppAttemptEventType.KILL)); assertEquals(RMAppAttemptState.FINAL_SAVING, @@ -999,6 +999,56 @@ public class TestRMAppAttemptTransitions { applicationAttempt.getAppAttemptState()); } + @Test(timeout = 1) + public void testAttemptRegisteredAtFailed() { +Container amContainer = allocateApplicationA
[hadoop] branch trunk updated: YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 6d7eedf YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda) 6d7eedf is described below commit 6d7eedfd28cc1712690db2f6ca8a281b0901ee28 Author: Wangda Tan AuthorDate: Thu Jan 17 14:20:10 2019 -0800 YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda) Change-Id: I4359f59a73a278a941f4bb9d106dd38c9cb471fe --- .../rmapp/attempt/RMAppAttemptImpl.java| 14 +++- .../rmapp/attempt/TestRMAppAttemptTransitions.java | 80 +- 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 3ec9c49..03039da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -437,9 +437,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { RMAppAttemptState.FAILED, EnumSet.of( RMAppAttemptEventType.LAUNCHED, + RMAppAttemptEventType.LAUNCH_FAILED, RMAppAttemptEventType.EXPIRE, RMAppAttemptEventType.KILL, RMAppAttemptEventType.FAIL, + RMAppAttemptEventType.REGISTERED, RMAppAttemptEventType.UNREGISTERED, RMAppAttemptEventType.STATUS_UPDATE, RMAppAttemptEventType.CONTAINER_ALLOCATED)) @@ -1203,10 +1205,16 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } // Set the masterContainer - appAttempt.setMasterContainer(amContainerAllocation.getContainers() - .get(0)); + Container amContainer = amContainerAllocation.getContainers().get(0); RMContainerImpl rmMasterContainer = (RMContainerImpl)appAttempt.scheduler - .getRMContainer(appAttempt.getMasterContainer().getId()); + .getRMContainer(amContainer.getId()); + //while one NM is removed, the scheduler will clean the container,the + //following CONTAINER_FINISHED event will handle the cleaned container. + //so just return RMAppAttemptState.SCHEDULED + if (rmMasterContainer == null) { +return RMAppAttemptState.SCHEDULED; + } + appAttempt.setMasterContainer(amContainer); rmMasterContainer.setAMContainer(true); // The node set in NMTokenSecrentManager is used for marking whether the // NMToken has been issued for this node to the AM. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 4a5c671..faecdb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -986,7 +986,7 @@ public class TestRMAppAttemptTransitions { public void testAttemptAddedAtFinalSaving() { submitApplicationAttempt(); -// SUBNITED->FINAL_SAVING +// SUBMITTED->FINAL_SAVING applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt .getAppAttemptId(), RMAppAttemptEventType.KILL)); assertEquals(RMAppAttemptState.FINAL_SAVING, @@ -999,6 +999,56 @@ public class TestRMAppAttemptTransitions { applicationAttempt.getAppAttemptState()); } + @Test(timeout = 1) + public void testAttemptRegisteredAtFailed() { +Container amContainer = allocateApplicationAttempt(); +launchApplicationAttempt(amContainer); + +//send CONTAINER_FINISHED event +NodeId any
[hadoop] annotated tag release-3.1.2-RC0 created (now ac9a81a)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to annotated tag release-3.1.2-RC0 in repository https://gitbox.apache.org/repos/asf/hadoop.git. at ac9a81a (tag) tagging 9f88af79029606b29a5d0ff9bea661a2a83a1dfb (commit) by Wangda Tan on Fri Jan 11 16:16:59 2019 -0800 - Log - Release candidate - 3.1.2-RC0 -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEETImYU83aTkDGAhK1s/plPVcwDUUFAlw5MfwACgkQs/plPVcw DUWUwg/9HZdjZV9i5teFuavrSOyguJ3ZJYqLzngAVVqln/66F/TxV511ScpYZInF nPmh2kFIGLuVMN4/FzAui04at0eonMroCrZ17wcu0zQh5evhy7LtS87ghPukhFOX RiAduDGauRHbSLuImERMIXH39S1sW48sRr1nBuJjFZjPJDkOVcerR0oYnt8ZX3P9 vy2Tb+B7xpq+8bkUxUJn0ysiUzqy0l8+UVA4iYJ9o3yWJ6WOTgbnqGU4e7s4F9Kv EdtBakiyvx8f0gS25LFs6LmS90pLtlxEQb+BY60cXxAWVfWf+sIHK2/Jsjq5A2LP +1lpR6gRtXmwTbXulWdqbpaZO2+k0bcR0O/3LfKAEQvgGA5DI8/5K+iiurXvLDLD CT5ZVUhMw9R//ZsPveKDPlm+wmDgqzK13SvfZZ/XSS0LouQ9/419oxenHR6Yp0WS dpycHqHFPa3ufxCSgN207p8YwsZihdxmGILCbCuzDQwWKncJTs6eZjYRuYAAUKCS ByWY1wHN6sDvhpvAWzQwwK9bOghH0yMJj+Shmm8prr4s0Hwh9hCi8S4BfIv3hz5v 3l9i6ohZA1Kwf9inRetLTkhFStIWLTUcFE78ycKJ9vc+VBK2y2RhCpSuzsHrj2in wCRLEjGvYzjB0U0NTkKVpPtu8aVPin6rcdCotk28cneapsCzUuk= =0WUH -END PGP SIGNATURE- --- No new revisions were added by this update. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.1.2 updated: YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.1.2 by this push: new e5c4ebb YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda) e5c4ebb is described below commit e5c4ebb8c94cfede5a437dab75519f5c7f93f7f0 Author: Wangda Tan AuthorDate: Mon Jan 7 12:30:30 2019 -0800 YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda) Change-Id: I41626a7b6f097d218d84e8497dd70b4b6d8f (cherry picked from commit 3570713a692e9985b51f534de50ce6d613f3b052) --- .../hadoop-yarn/conf/container-executor.cfg| 1 + .../apache/hadoop/yarn/conf/YarnConfiguration.java | 3 + .../linux/runtime/docker/DockerRunCommand.java | 5 + .../gpu/GpuDockerCommandPluginFactory.java | 4 + .../gpu/NvidiaDockerV2CommandPlugin.java | 111 ++ .../container-executor/impl/utils/docker-util.c| 20 .../container-executor/impl/utils/docker-util.h| 3 +- .../test/utils/test_docker_util.cc | 62 ++ .../linux/runtime/docker/TestDockerRunCommand.java | 5 +- .../gpu/TestNvidiaDockerV2CommandPlugin.java | 130 + .../src/site/markdown/DockerContainers.md | 1 + .../src/site/markdown/UsingGpus.md | 9 +- 12 files changed, 351 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg index d19874f..4df53df 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg +++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg @@ -16,6 +16,7 @@ feature.tc.enabled=false # docker.privileged-containers.enabled=false # docker.allowed.volume-drivers=## comma seperated list of allowed volume-drivers # docker.no-new-privileges.enabled=## enable/disable the no-new-privileges flag for docker run. Set to "true" to enable, disabled by default +# docker.allowed.runtimes=## comma seperated runtimes that can be used. # The configs below deal with settings for FPGA resource #[fpga] diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a5fb723..366b261 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1597,6 +1597,9 @@ public class YarnConfiguration extends Configuration { public static final String NVIDIA_DOCKER_V1 = "nvidia-docker-v1"; @Private + public static final String NVIDIA_DOCKER_V2 = "nvidia-docker-v2"; + + @Private public static final String DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL = NVIDIA_DOCKER_V1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index 241163d..2864e98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -165,6 +165,11 @@ public class DockerRunCommand extends DockerCommand { return this; } + public DockerRunCommand addRuntime(String runtime) { +super.addCommandArguments("runtime", runtime); +return this; + } + public DockerRunCommand groupAdd(String[] groups) { super.addCommandArguments("group-add", String.join(",", groups)); return this; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java index db4589a..051afd6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-se
[hadoop] branch branch-3.1 updated: YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.1 by this push: new 3570713 YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda) 3570713 is described below commit 3570713a692e9985b51f534de50ce6d613f3b052 Author: Wangda Tan AuthorDate: Mon Jan 7 12:30:30 2019 -0800 YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda) Change-Id: I41626a7b6f097d218d84e8497dd70b4b6d8f --- .../hadoop-yarn/conf/container-executor.cfg| 1 + .../apache/hadoop/yarn/conf/YarnConfiguration.java | 3 + .../linux/runtime/docker/DockerRunCommand.java | 5 + .../gpu/GpuDockerCommandPluginFactory.java | 4 + .../gpu/NvidiaDockerV2CommandPlugin.java | 111 ++ .../container-executor/impl/utils/docker-util.c| 20 .../container-executor/impl/utils/docker-util.h| 3 +- .../test/utils/test_docker_util.cc | 62 ++ .../linux/runtime/docker/TestDockerRunCommand.java | 5 +- .../gpu/TestNvidiaDockerV2CommandPlugin.java | 130 + .../src/site/markdown/DockerContainers.md | 1 + .../src/site/markdown/UsingGpus.md | 9 +- 12 files changed, 351 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg index d19874f..4df53df 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg +++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg @@ -16,6 +16,7 @@ feature.tc.enabled=false # docker.privileged-containers.enabled=false # docker.allowed.volume-drivers=## comma seperated list of allowed volume-drivers # docker.no-new-privileges.enabled=## enable/disable the no-new-privileges flag for docker run. Set to "true" to enable, disabled by default +# docker.allowed.runtimes=## comma seperated runtimes that can be used. # The configs below deal with settings for FPGA resource #[fpga] diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a5fb723..366b261 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1597,6 +1597,9 @@ public class YarnConfiguration extends Configuration { public static final String NVIDIA_DOCKER_V1 = "nvidia-docker-v1"; @Private + public static final String NVIDIA_DOCKER_V2 = "nvidia-docker-v2"; + + @Private public static final String DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL = NVIDIA_DOCKER_V1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index 241163d..2864e98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -165,6 +165,11 @@ public class DockerRunCommand extends DockerCommand { return this; } + public DockerRunCommand addRuntime(String runtime) { +super.addCommandArguments("runtime", runtime); +return this; + } + public DockerRunCommand groupAdd(String[] groups) { super.addCommandArguments("group-add", String.join(",", groups)); return this; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java index db4589a..051afd6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containerm
[hadoop] branch branch-3.2 updated: YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 1dc2b49 YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda) 1dc2b49 is described below commit 1dc2b49bfd74b61302cb2d771ce33fe939a393f7 Author: Wangda Tan AuthorDate: Mon Jan 7 12:21:33 2019 -0800 YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda) Change-Id: Ib8044307a4241f6b1b7b9b8266b9256f39b16384 --- .../hadoop-yarn/conf/container-executor.cfg| 1 + .../apache/hadoop/yarn/conf/YarnConfiguration.java | 3 + .../linux/runtime/docker/DockerRunCommand.java | 5 + .../gpu/GpuDockerCommandPluginFactory.java | 4 + .../gpu/NvidiaDockerV2CommandPlugin.java | 111 ++ .../container-executor/impl/utils/docker-util.c| 20 .../container-executor/impl/utils/docker-util.h| 3 +- .../test/utils/test_docker_util.cc | 62 ++ .../linux/runtime/docker/TestDockerRunCommand.java | 5 +- .../gpu/TestNvidiaDockerV2CommandPlugin.java | 130 + .../src/site/markdown/DockerContainers.md | 1 + .../src/site/markdown/UsingGpus.md | 9 +- 12 files changed, 351 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg index d19874f..4df53df 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg +++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg @@ -16,6 +16,7 @@ feature.tc.enabled=false # docker.privileged-containers.enabled=false # docker.allowed.volume-drivers=## comma seperated list of allowed volume-drivers # docker.no-new-privileges.enabled=## enable/disable the no-new-privileges flag for docker run. Set to "true" to enable, disabled by default +# docker.allowed.runtimes=## comma seperated runtimes that can be used. # The configs below deal with settings for FPGA resource #[fpga] diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index c5abec7..5f15931 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1636,6 +1636,9 @@ public class YarnConfiguration extends Configuration { public static final String NVIDIA_DOCKER_V1 = "nvidia-docker-v1"; @Private + public static final String NVIDIA_DOCKER_V2 = "nvidia-docker-v2"; + + @Private public static final String DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL = NVIDIA_DOCKER_V1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index 241163d..2864e98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -165,6 +165,11 @@ public class DockerRunCommand extends DockerCommand { return this; } + public DockerRunCommand addRuntime(String runtime) { +super.addCommandArguments("runtime", runtime); +return this; + } + public DockerRunCommand groupAdd(String[] groups) { super.addCommandArguments("group-add", String.join(",", groups)); return this; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java index db4589a..051afd6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containerm
[hadoop] branch trunk updated: YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 0a01d49 YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda) 0a01d49 is described below commit 0a01d499175569ffac9d3f31d980820ecac7e60b Author: Wangda Tan AuthorDate: Mon Jan 7 12:07:26 2019 -0800 YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda) Change-Id: Id8af27134d3286a7a10d85eda9be25df9689d0e7 --- .../hadoop-yarn/conf/container-executor.cfg| 1 + .../apache/hadoop/yarn/conf/YarnConfiguration.java | 3 + .../linux/runtime/docker/DockerRunCommand.java | 5 + .../gpu/GpuDockerCommandPluginFactory.java | 4 + .../gpu/NvidiaDockerV2CommandPlugin.java | 111 ++ .../container-executor/impl/utils/docker-util.c| 20 .../container-executor/impl/utils/docker-util.h| 3 +- .../test/utils/test_docker_util.cc | 62 ++ .../linux/runtime/docker/TestDockerRunCommand.java | 5 +- .../gpu/TestNvidiaDockerV2CommandPlugin.java | 130 + .../src/site/markdown/DockerContainers.md | 1 + .../src/site/markdown/UsingGpus.md | 9 +- 12 files changed, 351 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg index d19874f..4df53df 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg +++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg @@ -16,6 +16,7 @@ feature.tc.enabled=false # docker.privileged-containers.enabled=false # docker.allowed.volume-drivers=## comma seperated list of allowed volume-drivers # docker.no-new-privileges.enabled=## enable/disable the no-new-privileges flag for docker run. Set to "true" to enable, disabled by default +# docker.allowed.runtimes=## comma seperated runtimes that can be used. # The configs below deal with settings for FPGA resource #[fpga] diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 08f7f1a..c29707c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1665,6 +1665,9 @@ public class YarnConfiguration extends Configuration { public static final String NVIDIA_DOCKER_V1 = "nvidia-docker-v1"; @Private + public static final String NVIDIA_DOCKER_V2 = "nvidia-docker-v2"; + + @Private public static final String DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL = NVIDIA_DOCKER_V1; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index f4f3a9c..b0603a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -165,6 +165,11 @@ public class DockerRunCommand extends DockerCommand { return this; } + public DockerRunCommand addRuntime(String runtime) { +super.addCommandArguments("runtime", runtime); +return this; + } + public DockerRunCommand groupAdd(String[] groups) { super.addCommandArguments("group-add", String.join(",", groups)); return this; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java index db4589a..051afd6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourc
[hadoop] 01/01: Preparing for 3.1.2 release
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit 733d03485b019af02a4cbd3b60569ed2153fef4d Author: Wangda Tan AuthorDate: Mon Jan 7 10:09:01 2019 -0800 Preparing for 3.1.2 release Change-Id: I46544065b0352024c8b6a23e96a8e10170c925ba --- hadoop-assemblies/pom.xml | 4 ++-- hadoop-build-tools/pom.xml| 2 +- hadoop-client-modules/hadoop-client-api/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-check-invariants/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-integration-tests/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-minicluster/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-runtime/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client/pom.xml | 4 ++-- hadoop-client-modules/pom.xml | 2 +- hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml | 4 ++-- hadoop-cloud-storage-project/pom.xml | 4 ++-- hadoop-common-project/hadoop-annotations/pom.xml | 4 ++-- hadoop-common-project/hadoop-auth-examples/pom.xml| 4 ++-- hadoop-common-project/hadoop-auth/pom.xml | 4 ++-- hadoop-common-project/hadoop-common/pom.xml | 4 ++-- hadoop-common-project/hadoop-kms/pom.xml | 4 ++-- hadoop-common-project/hadoop-minikdc/pom.xml | 4 ++-- hadoop-common-project/hadoop-nfs/pom.xml | 4 ++-- hadoop-common-project/pom.xml | 4 ++-- hadoop-dist/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml| 4 ++-- hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml| 4 ++-- hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 ++-- hadoop-hdfs-project/pom.xml | 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml | 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml | 4 ++-- .../hadoop-mapreduce-client-hs-plugins/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml | 4 ++-- .../hadoop-mapreduce-client-nativetask/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml | 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml | 4 ++-- hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 4 ++-- hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml| 4 ++-- hadoop-mapreduce-project/pom.xml | 4 ++-- hadoop-maven-plugins/pom.xml | 2 +- hadoop-minicluster/pom.xml| 4 ++-- hadoop-project-dist/pom.xml | 4 ++-- hadoop-project/pom.xml| 4 ++-- hadoop-tools/hadoop-aliyun/pom.xml| 2 +- hadoop-tools/hadoop-archive-logs/pom.xml | 4 ++-- hadoop-tools/hadoop-archives/pom.xml | 4 ++-- hadoop-tools/hadoop-aws/pom.xml | 4 ++-- hadoop-tools/hadoop-azure-datalake/pom.xml| 2 +- hadoop-tools/hadoop-azure/pom.xml | 2 +- hadoop-tools/hadoop-datajoin/pom.xml | 4 ++-- hadoop-tools/hadoop-distcp/pom.xml| 4 ++-- hadoop-tools/hadoop-extras/pom.xml| 4 ++-- hadoop-tools/hadoop-fs2img/pom.xml| 4 ++-- hadoop-tools/hadoop-gridmix/pom.xml | 4 ++-- hadoop-tools/hadoop-kafka/pom.xml | 4 ++-- hadoop-tools/hadoop-openstack/pom.xml | 4 ++-- hadoop-tools/hadoop-pipes/pom.xml
[hadoop] branch branch-3.1.2 created (now 733d034)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to branch branch-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git. at 733d034 Preparing for 3.1.2 release This branch includes the following new commits: new 733d034 Preparing for 3.1.2 release The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.1 updated: Preparing for 3.1.3 development
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.1 by this push: new 31ea2f7 Preparing for 3.1.3 development 31ea2f7 is described below commit 31ea2f780631d40478e3ce9d6fef70832081f720 Author: Wangda Tan AuthorDate: Mon Jan 7 10:04:58 2019 -0800 Preparing for 3.1.3 development Change-Id: I3c3d3ee47dc4fef239127b4452ff14676fa26e3d --- hadoop-assemblies/pom.xml | 4 ++-- hadoop-build-tools/pom.xml| 2 +- hadoop-client-modules/hadoop-client-api/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-check-invariants/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-integration-tests/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-minicluster/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-runtime/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client/pom.xml | 4 ++-- hadoop-client-modules/pom.xml | 2 +- hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml | 4 ++-- hadoop-cloud-storage-project/pom.xml | 4 ++-- hadoop-common-project/hadoop-annotations/pom.xml | 4 ++-- hadoop-common-project/hadoop-auth-examples/pom.xml| 4 ++-- hadoop-common-project/hadoop-auth/pom.xml | 4 ++-- hadoop-common-project/hadoop-common/pom.xml | 4 ++-- hadoop-common-project/hadoop-kms/pom.xml | 4 ++-- hadoop-common-project/hadoop-minikdc/pom.xml | 4 ++-- hadoop-common-project/hadoop-nfs/pom.xml | 4 ++-- hadoop-common-project/pom.xml | 4 ++-- hadoop-dist/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml| 4 ++-- hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml| 4 ++-- hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 ++-- hadoop-hdfs-project/pom.xml | 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml | 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml | 4 ++-- .../hadoop-mapreduce-client-hs-plugins/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml | 4 ++-- .../hadoop-mapreduce-client-nativetask/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml | 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml | 4 ++-- hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 4 ++-- hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml| 4 ++-- hadoop-mapreduce-project/pom.xml | 4 ++-- hadoop-maven-plugins/pom.xml | 2 +- hadoop-minicluster/pom.xml| 4 ++-- hadoop-project-dist/pom.xml | 4 ++-- hadoop-project/pom.xml| 4 ++-- hadoop-tools/hadoop-aliyun/pom.xml| 2 +- hadoop-tools/hadoop-archive-logs/pom.xml | 4 ++-- hadoop-tools/hadoop-archives/pom.xml | 4 ++-- hadoop-tools/hadoop-aws/pom.xml | 4 ++-- hadoop-tools/hadoop-azure-datalake/pom.xml| 2 +- hadoop-tools/hadoop-azure/pom.xml | 2 +- hadoop-tools/hadoop-datajoin/pom.xml | 4 ++-- hadoop-tools/hadoop-distcp/pom.xml| 4 ++-- hadoop-tools/hadoop-extras/pom.xml| 4 ++-- hadoop-tools/hadoop-fs2img/pom.xml| 4 ++-- hadoop-tools/hadoop-gridmix/pom.xml | 4 ++-- hadoop-tools/hadoop-kafka/pom.xml
[hadoop] branch branch-3.2 updated (71bee05 -> cffe5c1)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 71bee05 Revert "HADOOP-15759. AliyunOSS: Update oss-sdk version to 3.0.0. Contributed by Jinhu Wu." new 1ad1ab5 YARN-9141. [Submarine] JobStatus outputs with system UTC clock, not local clock. (Zac Zhou via wangda) new cffe5c1 YARN-9160. [Submarine] Document 'PYTHONPATH' environment variable setting when using -localization options. (Zhankun Tang via wangda) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../hadoop/yarn/submarine/common/api/JobStatus.java| 4 ++-- .../src/site/markdown/QuickStart.md| 18 ++ 2 files changed, 20 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 02/02: YARN-9160. [Submarine] Document 'PYTHONPATH' environment variable setting when using -localization options. (Zhankun Tang via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit cffe5c1ba089cb6a720f479a40a8e300127caa44 Author: Wangda Tan AuthorDate: Sun Jan 6 10:25:35 2019 -0800 YARN-9160. [Submarine] Document 'PYTHONPATH' environment variable setting when using -localization options. (Zhankun Tang via wangda) Change-Id: I115b6710eb11501e21ab3e9e7cd1a87f322e5cf8 (cherry picked from commit 2c02aa6ec259128934cc5468cf66104a624d88a7) --- .../src/site/markdown/QuickStart.md| 18 ++ 1 file changed, 18 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md index 8e7a956..21db6bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md @@ -87,6 +87,24 @@ usage: job run -localization "/user/yarn/mydir3:/opt/mydir3" -localization "./mydir1:." ``` + + Notes: +When using `localization` option to make a collection of dependency Python +scripts available to entry python script in the container, you may also need to + set `PYTHONPATH` environment variable as below to avoid module import error +reported from `entry_script.py`. + +``` +... job run + # the entry point + --localization entry_script.py:/entry_script.py + # the dependency Python scripts of the entry point + --localization other_scripts_dir:/other_scripts_dir + # the PYTHONPATH env to make dependency available to entry script + --env PYTHONPATH="/other_scripts_dir" + --worker_launch_cmd "python /entry_script.py ..." +``` + ### Submarine Configuration For submarine internal configuration, please create a `submarine.xml` which should be placed under `$HADOOP_CONF_DIR`. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 01/02: YARN-9141. [Submarine] JobStatus outputs with system UTC clock, not local clock. (Zac Zhou via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit 1ad1ab5d7a2a292039ccb06ae659f691e9ef63aa Author: Wangda Tan AuthorDate: Sun Jan 6 10:24:29 2019 -0800 YARN-9141. [Submarine] JobStatus outputs with system UTC clock, not local clock. (Zac Zhou via wangda) Change-Id: Ib2b17cf7860ddc9188f06498086de3326bc09519 (cherry picked from commit 8f004fe4e0d479e26d0063427566edc1606bced5) --- .../java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java index 6e390f3..0ba9de9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java @@ -18,7 +18,7 @@ package org.apache.hadoop.yarn.submarine.common.api; import java.io.PrintStream; -import java.time.Instant; +import java.time.LocalDateTime; import java.util.List; /** @@ -34,7 +34,7 @@ public class JobStatus { public void nicePrint(PrintStream out) { out.println( "Job Name=" + this.jobName + ", status=" + state.name() + " time=" -+ Instant.now()); ++ LocalDateTime.now()); if (JobState.isFinal(this.state)) { return; } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 01/02: YARN-9141. [Submarine] JobStatus outputs with system UTC clock, not local clock. (Zac Zhou via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git commit 8f004fe4e0d479e26d0063427566edc1606bced5 Author: Wangda Tan AuthorDate: Sun Jan 6 10:24:29 2019 -0800 YARN-9141. [Submarine] JobStatus outputs with system UTC clock, not local clock. (Zac Zhou via wangda) Change-Id: Ib2b17cf7860ddc9188f06498086de3326bc09519 --- .../java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java index 6e390f3..0ba9de9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java @@ -18,7 +18,7 @@ package org.apache.hadoop.yarn.submarine.common.api; import java.io.PrintStream; -import java.time.Instant; +import java.time.LocalDateTime; import java.util.List; /** @@ -34,7 +34,7 @@ public class JobStatus { public void nicePrint(PrintStream out) { out.println( "Job Name=" + this.jobName + ", status=" + state.name() + " time=" -+ Instant.now()); ++ LocalDateTime.now()); if (JobState.isFinal(this.state)) { return; } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 02/02: YARN-9160. [Submarine] Document 'PYTHONPATH' environment variable setting when using -localization options. (Zhankun Tang via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git commit 2c02aa6ec259128934cc5468cf66104a624d88a7 Author: Wangda Tan AuthorDate: Sun Jan 6 10:25:35 2019 -0800 YARN-9160. [Submarine] Document 'PYTHONPATH' environment variable setting when using -localization options. (Zhankun Tang via wangda) Change-Id: I115b6710eb11501e21ab3e9e7cd1a87f322e5cf8 --- .../src/site/markdown/QuickStart.md| 18 ++ 1 file changed, 18 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md index 8e7a956..21db6bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md @@ -87,6 +87,24 @@ usage: job run -localization "/user/yarn/mydir3:/opt/mydir3" -localization "./mydir1:." ``` + + Notes: +When using `localization` option to make a collection of dependency Python +scripts available to entry python script in the container, you may also need to + set `PYTHONPATH` environment variable as below to avoid module import error +reported from `entry_script.py`. + +``` +... job run + # the entry point + --localization entry_script.py:/entry_script.py + # the dependency Python scripts of the entry point + --localization other_scripts_dir:/other_scripts_dir + # the PYTHONPATH env to make dependency available to entry script + --env PYTHONPATH="/other_scripts_dir" + --worker_launch_cmd "python /entry_script.py ..." +``` + ### Submarine Configuration For submarine internal configuration, please create a `submarine.xml` which should be placed under `$HADOOP_CONF_DIR`. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (650b9cb -> 2c02aa6)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 650b9cb YARN-9178. TestRMAdminCli#testHelp is failing in trunk. Contributed by Abhishek Modi. new 8f004fe YARN-9141. [Submarine] JobStatus outputs with system UTC clock, not local clock. (Zac Zhou via wangda) new 2c02aa6 YARN-9160. [Submarine] Document 'PYTHONPATH' environment variable setting when using -localization options. (Zhankun Tang via wangda) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../hadoop/yarn/submarine/common/api/JobStatus.java| 4 ++-- .../src/site/markdown/QuickStart.md| 18 ++ 2 files changed, 20 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[3/3] hadoop git commit: YARN-9112. [Submarine] Support polling applicationId when it's not ready in cluster. (Zhankun Tang via wangda)
YARN-9112. [Submarine] Support polling applicationId when it's not ready in cluster. (Zhankun Tang via wangda) Change-Id: I73d73f3d631b28fb9866faa56571839b13824a97 (cherry picked from commit 9fba6cc2471db8df9ca6da7b9df378e3072dfeb2) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29e4e5f6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29e4e5f6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29e4e5f6 Branch: refs/heads/branch-3.2 Commit: 29e4e5f62c3cd3a2bbd3160d8721f518301765e0 Parents: ef0f835 Author: Wangda Tan Authored: Wed Dec 12 11:44:58 2018 -0800 Committer: Wangda Tan Committed: Wed Dec 12 11:52:16 2018 -0800 -- .../yarnservice/YarnServiceJobSubmitter.java| 24 1 file changed, 20 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/29e4e5f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java index 8cc9794..b92438d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java @@ -866,10 +866,26 @@ public class YarnServiceJobSubmitter implements JobSubmitter { } String appStatus=appAdminClient.getStatusString(serviceSpec.getName()); -Service app=ServiceApiUtil.jsonSerDeser.fromJson(appStatus); -if(app.getId() == null) { - throw new YarnException("Can't get application id for Service " + - serviceSpec.getName()); +Service app = ServiceApiUtil.jsonSerDeser.fromJson(appStatus); + +// Retry multiple times if applicationId is null +int maxRetryTimes = 30; +int count = 0; +while (app.getId() == null && count < maxRetryTimes) { + LOG.info("Waiting for application Id. AppStatusString=\n {}", appStatus); + try { +Thread.sleep(1000); + } catch (InterruptedException e) { +throw new IOException(e); + } + appStatus = appAdminClient.getStatusString(serviceSpec.getName()); + app = ServiceApiUtil.jsonSerDeser.fromJson(appStatus); + count++; +} +// Retry timeout +if (app.getId() == null) { + throw new YarnException( + "Can't get application id for Service " + serviceSpec.getName()); } ApplicationId appid = ApplicationId.fromString(app.getId()); appAdminClient.stop(); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/3] hadoop git commit: YARN-8714. [Submarine] Support files/tarballs to be localized for a training job. (Zhankun Tang via wangda)
YARN-8714. [Submarine] Support files/tarballs to be localized for a training job. (Zhankun Tang via wangda) Change-Id: I845131273e52a9d81dbc813ea6d4af06b205e334 (cherry picked from commit c771fe6e10bd3f8533a01026de3c33db2a983cf4) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef0f8355 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef0f8355 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef0f8355 Branch: refs/heads/branch-3.2 Commit: ef0f8355372652a55bf1680ddccab0012f55a283 Parents: bbe1ddd Author: Wangda Tan Authored: Wed Dec 12 11:43:23 2018 -0800 Committer: Wangda Tan Committed: Wed Dec 12 11:52:07 2018 -0800 -- .../yarn/submarine/client/cli/CliConstants.java | 1 + .../yarn/submarine/client/cli/RunJobCli.java| 18 + .../client/cli/param/Localization.java | 133 .../client/cli/param/RunJobParameters.java | 15 + .../client/cli/param/package-info.java | 19 + .../common/conf/SubmarineConfiguration.java | 15 + .../fs/DefaultRemoteDirectoryManager.java | 70 +- .../common/fs/RemoteDirectoryManager.java | 18 +- .../common/FSBasedSubmarineStorageImpl.java | 8 +- .../yarnservice/YarnServiceJobSubmitter.java| 259 +- .../src/site/markdown/QuickStart.md | 26 + .../yarnservice/TestYarnServiceRunJobCli.java | 795 ++- .../submarine/common/MockClientContext.java | 8 +- .../common/fs/MockRemoteDirectoryManager.java | 90 ++- 14 files changed, 1433 insertions(+), 42 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef0f8355/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java index 2d7a472..da4253b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java @@ -52,6 +52,7 @@ public class CliConstants { public static final String QUICKLINK = "quicklink"; public static final String TENSORBOARD_DOCKER_IMAGE = "tensorboard_docker_image"; + public static final String LOCALIZATION = "localization"; public static final String KEYTAB = "keytab"; public static final String PRINCIPAL = "principal"; public static final String DISTRIBUTE_KEYTAB = "distribute_keytab"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef0f8355/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java index fc57be9..b764d6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java @@ -125,6 +125,24 @@ public class RunJobCli extends AbstractCli { + "if want to link to first worker's 7070 port, and text of quicklink " + "is Notebook_UI, user need to specify --quicklink " + "Notebook_UI=https://master-0:7070";); +options.addOption(CliConstants.LOCALIZATION, true, "Specify" ++ " localization to make remote/local file/directory available to" ++ " all container(Docker)." ++ " Argument format is \"RemoteUri:LocalFilePath[:rw] \" (ro" ++ " permission is not supported yet)" ++ " The RemoteUri can be a file or directory in local or" ++ " HDFS or s3 or abfs or http .etc." ++ " The LocalFilePath can be
[1/3] hadoop git commit: YARN-8714. [Submarine] Support files/tarballs to be localized for a training job. (Zhankun Tang via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.2 bbe1ddd30 -> 29e4e5f62 http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef0f8355/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java index b637036..4334293 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java @@ -19,7 +19,9 @@ package org.apache.hadoop.yarn.submarine.common.fs; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import java.io.File; @@ -29,6 +31,7 @@ public class MockRemoteDirectoryManager implements RemoteDirectoryManager { private File jobsParentDir = null; private File modelParentDir = null; + private File jobDir = null; @Override public Path getJobStagingArea(String jobName, boolean create) throws IOException { @@ -41,10 +44,11 @@ public class MockRemoteDirectoryManager implements RemoteDirectoryManager { } } -File jobDir = new File(jobsParentDir.getAbsolutePath(), jobName); +this.jobDir = new File(jobsParentDir.getAbsolutePath(), jobName); if (create && !jobDir.exists()) { if (!jobDir.mkdirs()) { -throw new IOException("Failed to mkdirs for " + jobDir.getAbsolutePath()); +throw new IOException("Failed to mkdirs for " ++ jobDir.getAbsolutePath()); } } return new Path(jobDir.getAbsolutePath()); @@ -57,7 +61,8 @@ public class MockRemoteDirectoryManager implements RemoteDirectoryManager { } @Override - public Path getModelDir(String modelName, boolean create) throws IOException { + public Path getModelDir(String modelName, boolean create) + throws IOException { if (modelParentDir == null && create) { modelParentDir = new File( "target/_models_" + System.currentTimeMillis()); @@ -70,19 +75,94 @@ public class MockRemoteDirectoryManager implements RemoteDirectoryManager { File modelDir = new File(modelParentDir.getAbsolutePath(), modelName); if (create) { if (!modelDir.exists() && !modelDir.mkdirs()) { -throw new IOException("Failed to mkdirs for " + modelDir.getAbsolutePath()); +throw new IOException("Failed to mkdirs for " ++ modelDir.getAbsolutePath()); } } return new Path(modelDir.getAbsolutePath()); } @Override - public FileSystem getFileSystem() throws IOException { + public FileSystem getDefaultFileSystem() throws IOException { return FileSystem.getLocal(new Configuration()); } @Override + public FileSystem getFileSystemByUri(String uri) throws IOException { +return getDefaultFileSystem(); + } + + @Override public Path getUserRootFolder() throws IOException { return new Path("s3://generated_root_dir"); } + + @Override + public boolean isDir(String uri) throws IOException { +return getDefaultFileSystem().getFileStatus( +new Path(convertToStagingPath(uri))).isDirectory(); + + } + + @Override + public boolean isRemote(String uri) throws IOException { +String scheme = new Path(uri).toUri().getScheme(); +if (null == scheme) { + return false; +} +return !scheme.startsWith("file://"); + } + + private String convertToStagingPath(String uri) throws IOException { +String ret = uri; +if (isRemote(uri)) { + String dirName = new Path(uri).getName(); + ret = this.jobDir.getAbsolutePath() + + "/" + dirName; +} +return ret; + } + + /** + * We use staging dir as mock HDFS dir. + * */ + @Override + public boolean copyRemoteToLocal(String remoteUri, String localUri) + throws IOException { +// mock the copy from HDFS into a local copy +Path remoteToLocalDir = new Path(convertToStagingPath(remoteUri)); +File old = new File(convertToStagingPath(localUri)); +if (old.isDirectory() && old.exists()) { + if (!FileUtil.fullyDelete(old)) { +throw new IOException("Cannot delete temp dir:" ++ old.getAbsolutePath()); + } +} +return FileUtil.copy(getDefaultFileSystem(), remoteToLocalDir,
[2/6] hadoop git commit: YARN-8714. [Submarine] Support files/tarballs to be localized for a training job. (Zhankun Tang via wangda)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c771fe6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java index b637036..4334293 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java @@ -19,7 +19,9 @@ package org.apache.hadoop.yarn.submarine.common.fs; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import java.io.File; @@ -29,6 +31,7 @@ public class MockRemoteDirectoryManager implements RemoteDirectoryManager { private File jobsParentDir = null; private File modelParentDir = null; + private File jobDir = null; @Override public Path getJobStagingArea(String jobName, boolean create) throws IOException { @@ -41,10 +44,11 @@ public class MockRemoteDirectoryManager implements RemoteDirectoryManager { } } -File jobDir = new File(jobsParentDir.getAbsolutePath(), jobName); +this.jobDir = new File(jobsParentDir.getAbsolutePath(), jobName); if (create && !jobDir.exists()) { if (!jobDir.mkdirs()) { -throw new IOException("Failed to mkdirs for " + jobDir.getAbsolutePath()); +throw new IOException("Failed to mkdirs for " ++ jobDir.getAbsolutePath()); } } return new Path(jobDir.getAbsolutePath()); @@ -57,7 +61,8 @@ public class MockRemoteDirectoryManager implements RemoteDirectoryManager { } @Override - public Path getModelDir(String modelName, boolean create) throws IOException { + public Path getModelDir(String modelName, boolean create) + throws IOException { if (modelParentDir == null && create) { modelParentDir = new File( "target/_models_" + System.currentTimeMillis()); @@ -70,19 +75,94 @@ public class MockRemoteDirectoryManager implements RemoteDirectoryManager { File modelDir = new File(modelParentDir.getAbsolutePath(), modelName); if (create) { if (!modelDir.exists() && !modelDir.mkdirs()) { -throw new IOException("Failed to mkdirs for " + modelDir.getAbsolutePath()); +throw new IOException("Failed to mkdirs for " ++ modelDir.getAbsolutePath()); } } return new Path(modelDir.getAbsolutePath()); } @Override - public FileSystem getFileSystem() throws IOException { + public FileSystem getDefaultFileSystem() throws IOException { return FileSystem.getLocal(new Configuration()); } @Override + public FileSystem getFileSystemByUri(String uri) throws IOException { +return getDefaultFileSystem(); + } + + @Override public Path getUserRootFolder() throws IOException { return new Path("s3://generated_root_dir"); } + + @Override + public boolean isDir(String uri) throws IOException { +return getDefaultFileSystem().getFileStatus( +new Path(convertToStagingPath(uri))).isDirectory(); + + } + + @Override + public boolean isRemote(String uri) throws IOException { +String scheme = new Path(uri).toUri().getScheme(); +if (null == scheme) { + return false; +} +return !scheme.startsWith("file://"); + } + + private String convertToStagingPath(String uri) throws IOException { +String ret = uri; +if (isRemote(uri)) { + String dirName = new Path(uri).getName(); + ret = this.jobDir.getAbsolutePath() + + "/" + dirName; +} +return ret; + } + + /** + * We use staging dir as mock HDFS dir. + * */ + @Override + public boolean copyRemoteToLocal(String remoteUri, String localUri) + throws IOException { +// mock the copy from HDFS into a local copy +Path remoteToLocalDir = new Path(convertToStagingPath(remoteUri)); +File old = new File(convertToStagingPath(localUri)); +if (old.isDirectory() && old.exists()) { + if (!FileUtil.fullyDelete(old)) { +throw new IOException("Cannot delete temp dir:" ++ old.getAbsolutePath()); + } +} +return FileUtil.copy(getDefaultFileSystem(), remoteToLocalDir, +new File(localUri), false, +getDefaultFileSystem().getConf()); + } +
[6/6] hadoop git commit: YARN-8885. [DevicePlugin] Support NM APIs to query device resource allocation. (Zhankun Tang via wangda)
YARN-8885. [DevicePlugin] Support NM APIs to query device resource allocation. (Zhankun Tang via wangda) Change-Id: I2a9870709b512af1ac6c09c9701d0b3c0791ff32 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37eb919c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37eb919c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37eb919c Branch: refs/heads/trunk Commit: 37eb919c59d10955213f5b8230e5687887f1f9c3 Parents: 9fba6cc Author: Wangda Tan Authored: Wed Dec 12 11:45:47 2018 -0800 Committer: Wangda Tan Committed: Wed Dec 12 11:45:47 2018 -0800 -- .../deviceframework/AssignedDevice.java | 78 .../deviceframework/DevicePluginAdapter.java| 18 - .../webapp/dao/NMDeviceResourceInfo.java| 56 ++ .../TestDevicePluginAdapter.java| 67 + 4 files changed, 218 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/37eb919c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/AssignedDevice.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/AssignedDevice.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/AssignedDevice.java new file mode 100644 index 000..cad95ab --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/AssignedDevice.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device; + +import java.io.Serializable; +import java.util.Objects; + +/** + * Device wrapper class used for NM REST API. + * */ +public class AssignedDevice implements Serializable, Comparable { + + private static final long serialVersionUID = -544285507952217366L; + + private Device device; + private String containerId; + + public AssignedDevice(ContainerId cId, Device dev) { +this.device = dev; +this.containerId = cId.toString(); + } + + public Device getDevice() { +return device; + } + + public String getContainerId() { +return containerId; + } + + @Override + public int compareTo(Object o) { +if (o == null || !(o instanceof AssignedDevice)) { + return -1; +} +AssignedDevice other = (AssignedDevice) o; +int result = getDevice().compareTo(other.getDevice()); +if (0 != result) { + return result; +} +return getContainerId().compareTo(other.getContainerId()); + } + + @Override + public boolean equals(Object o) { +if (o == null || !(o instanceof AssignedDevice)) { + return false; +} +AssignedDevice other = (AssignedDevice) o; +return getDevice().equals(other.getDevice()) +&& getContainerId().equals(other.getContainerId()); + } + + @Override + public int hashCode() { +return Objects.hash(getDevice(), getContainerId()); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/37eb919c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-ya
[4/6] hadoop git commit: YARN-9015. [DevicePlugin] Add an interface for device plugin to provide customized scheduler. (Zhankun Tang via wangda)
YARN-9015. [DevicePlugin] Add an interface for device plugin to provide customized scheduler. (Zhankun Tang via wangda) Change-Id: Ib2e4ae47a6f29bb3082c1f8520cf5a52ca720979 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61bdcb7b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61bdcb7b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61bdcb7b Branch: refs/heads/trunk Commit: 61bdcb7b2bc3057a8fe2b9dfb2b8154a585eab87 Parents: c771fe6 Author: Wangda Tan Authored: Wed Dec 12 11:44:22 2018 -0800 Committer: Wangda Tan Committed: Wed Dec 12 11:44:22 2018 -0800 -- .../api/deviceplugin/DevicePluginScheduler.java | 38 ++ .../resourceplugin/ResourcePluginManager.java | 29 +++- .../deviceframework/DeviceMappingManager.java | 73 +--- .../deviceframework/DevicePluginAdapter.java| 2 +- .../TestResourcePluginManager.java | 28 .../deviceframework/FakeTestDevicePlugin1.java | 18 - .../deviceframework/FakeTestDevicePlugin5.java | 56 +++ .../TestDevicePluginAdapter.java| 70 ++- 8 files changed, 300 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/61bdcb7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePluginScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePluginScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePluginScheduler.java new file mode 100644 index 000..80ffdf7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePluginScheduler.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin; + +import java.util.Set; + +/** + * An optional interface to implement if custom device scheduling is needed. + * If this is not implemented, the device framework will do scheduling. + * */ +public interface DevicePluginScheduler { + /** + * Called when allocating devices. The framework will do all device book + * keeping and fail recovery. So this hook could be stateless and only do + * scheduling based on available devices passed in. It could be + * invoked multiple times by the framework. + * @param availableDevices Devices allowed to be chosen from. + * @param count Number of device to be allocated. + * @return A set of {@link Device} allocated + * */ + Set allocateDevices(Set availableDevices, int count); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/61bdcb7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java index bd47cfe..0bc241d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java +++ b/hadoop-yarn-projec
[3/6] hadoop git commit: YARN-8714. [Submarine] Support files/tarballs to be localized for a training job. (Zhankun Tang via wangda)
YARN-8714. [Submarine] Support files/tarballs to be localized for a training job. (Zhankun Tang via wangda) Change-Id: I845131273e52a9d81dbc813ea6d4af06b205e334 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c771fe6e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c771fe6e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c771fe6e Branch: refs/heads/trunk Commit: c771fe6e10bd3f8533a01026de3c33db2a983cf4 Parents: 8a53611 Author: Wangda Tan Authored: Wed Dec 12 11:43:23 2018 -0800 Committer: Wangda Tan Committed: Wed Dec 12 11:43:23 2018 -0800 -- .../yarn/submarine/client/cli/CliConstants.java | 1 + .../yarn/submarine/client/cli/RunJobCli.java| 18 + .../client/cli/param/Localization.java | 133 .../client/cli/param/RunJobParameters.java | 15 + .../client/cli/param/package-info.java | 19 + .../common/conf/SubmarineConfiguration.java | 15 + .../fs/DefaultRemoteDirectoryManager.java | 70 +- .../common/fs/RemoteDirectoryManager.java | 18 +- .../common/FSBasedSubmarineStorageImpl.java | 8 +- .../yarnservice/YarnServiceJobSubmitter.java| 259 +- .../src/site/markdown/QuickStart.md | 26 + .../yarnservice/TestYarnServiceRunJobCli.java | 795 ++- .../submarine/common/MockClientContext.java | 8 +- .../common/fs/MockRemoteDirectoryManager.java | 90 ++- 14 files changed, 1433 insertions(+), 42 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c771fe6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java index 2d7a472..da4253b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java @@ -52,6 +52,7 @@ public class CliConstants { public static final String QUICKLINK = "quicklink"; public static final String TENSORBOARD_DOCKER_IMAGE = "tensorboard_docker_image"; + public static final String LOCALIZATION = "localization"; public static final String KEYTAB = "keytab"; public static final String PRINCIPAL = "principal"; public static final String DISTRIBUTE_KEYTAB = "distribute_keytab"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c771fe6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java index fc57be9..b764d6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java @@ -125,6 +125,24 @@ public class RunJobCli extends AbstractCli { + "if want to link to first worker's 7070 port, and text of quicklink " + "is Notebook_UI, user need to specify --quicklink " + "Notebook_UI=https://master-0:7070";); +options.addOption(CliConstants.LOCALIZATION, true, "Specify" ++ " localization to make remote/local file/directory available to" ++ " all container(Docker)." ++ " Argument format is \"RemoteUri:LocalFilePath[:rw] \" (ro" ++ " permission is not supported yet)" ++ " The RemoteUri can be a file or directory in local or" ++ " HDFS or s3 or abfs or http .etc." ++ " The LocalFilePath can be absolute or relative." ++ " If it's a relative path, it'
[1/6] hadoop git commit: YARN-9078. [Submarine] Clean up the code of CliUtils#parseResourcesString. (Zhankun Tang via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 561b61cfb -> 37eb919c5 YARN-9078. [Submarine] Clean up the code of CliUtils#parseResourcesString. (Zhankun Tang via wangda) Change-Id: I21ac5757b6115d55ec3157ba25db1b65bc85a37b Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a536111 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a536111 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a536111 Branch: refs/heads/trunk Commit: 8a5361116e6158b28b185ba903ece4aa454344c8 Parents: 561b61c Author: Wangda Tan Authored: Wed Dec 12 11:42:44 2018 -0800 Committer: Wangda Tan Committed: Wed Dec 12 11:42:44 2018 -0800 -- .../yarn/submarine/client/cli/CliUtils.java | 23 +++- 1 file changed, 8 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a536111/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java index 588a9e6..f3eee7c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java @@ -74,19 +74,10 @@ public class CliUtils { return newCli; } - // TODO, this duplicated to Client of distributed shell, should cleanup private static Map parseResourcesString(String resourcesStr) { Map resources = new HashMap<>(); - -// Ignore the grouping "[]" -if (resourcesStr.startsWith("[")) { - resourcesStr = resourcesStr.substring(1); -} -if (resourcesStr.endsWith("]")) { - resourcesStr = resourcesStr.substring(0, resourcesStr.length() - 1); -} - -for (String resource : resourcesStr.trim().split(",")) { +String[] pairs = resourcesStr.trim().split(","); +for (String resource : pairs) { resource = resource.trim(); if (!resource.matches(RES_PATTERN)) { throw new IllegalArgumentException("\"" + resource + "\" is not a " @@ -97,8 +88,9 @@ public class CliUtils { String key = splits[0], value = splits[1]; String units = ResourceUtils.getUnits(value); - String valueWithoutUnit = value.substring(0, value.length() - units.length()).trim(); - Long resourceValue = Long.valueOf(valueWithoutUnit); + String valueWithoutUnit = value.substring(0, + value.length()- units.length()).trim(); + long resourceValue = Long.parseLong(valueWithoutUnit); // Convert commandline unit to standard YARN unit. if (units.equals("M") || units.equals("m")) { @@ -107,7 +99,7 @@ public class CliUtils { units = "Gi"; } else if (units.isEmpty()) { // do nothing; - } else{ + } else { throw new IllegalArgumentException("Acceptable units are M/G or empty"); } @@ -121,7 +113,8 @@ public class CliUtils { if (key.equals("memory")) { key = ResourceInformation.MEMORY_URI; -resourceValue = UnitsConversionUtil.convert(units, "Mi", resourceValue); +resourceValue = UnitsConversionUtil.convert(units, "Mi", +resourceValue); } // special handle gpu - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[5/6] hadoop git commit: YARN-9112. [Submarine] Support polling applicationId when it's not ready in cluster. (Zhankun Tang via wangda)
YARN-9112. [Submarine] Support polling applicationId when it's not ready in cluster. (Zhankun Tang via wangda) Change-Id: I73d73f3d631b28fb9866faa56571839b13824a97 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9fba6cc2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9fba6cc2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9fba6cc2 Branch: refs/heads/trunk Commit: 9fba6cc2471db8df9ca6da7b9df378e3072dfeb2 Parents: 61bdcb7 Author: Wangda Tan Authored: Wed Dec 12 11:44:58 2018 -0800 Committer: Wangda Tan Committed: Wed Dec 12 11:44:58 2018 -0800 -- .../yarnservice/YarnServiceJobSubmitter.java| 24 1 file changed, 20 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fba6cc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java index 496ba7c..06d33b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java @@ -865,10 +865,26 @@ public class YarnServiceJobSubmitter implements JobSubmitter { } String appStatus=appAdminClient.getStatusString(serviceSpec.getName()); -Service app=ServiceApiUtil.jsonSerDeser.fromJson(appStatus); -if(app.getId() == null) { - throw new YarnException("Can't get application id for Service " + - serviceSpec.getName()); +Service app = ServiceApiUtil.jsonSerDeser.fromJson(appStatus); + +// Retry multiple times if applicationId is null +int maxRetryTimes = 30; +int count = 0; +while (app.getId() == null && count < maxRetryTimes) { + LOG.info("Waiting for application Id. AppStatusString=\n {}", appStatus); + try { +Thread.sleep(1000); + } catch (InterruptedException e) { +throw new IOException(e); + } + appStatus = appAdminClient.getStatusString(serviceSpec.getName()); + app = ServiceApiUtil.jsonSerDeser.fromJson(appStatus); + count++; +} +// Retry timeout +if (app.getId() == null) { + throw new YarnException( + "Can't get application id for Service " + serviceSpec.getName()); } ApplicationId appid = ApplicationId.fromString(app.getId()); appAdminClient.stop(); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)" This reverts commit 920401b2966c2dcd7cd1b3dfd7a764f08412323f. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/829b530a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/829b530a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/829b530a Branch: refs/heads/branch-3.2.0 Commit: 829b530a082307f9c275f4e9e4a1ee35aefbf4d6 Parents: 542b224 Author: Wangda Tan Authored: Tue Dec 4 14:14:41 2018 -0800 Committer: Wangda Tan Committed: Tue Dec 4 14:14:41 2018 -0800 -- .../resources/assemblies/hadoop-yarn-dist.xml | 8 - .../installation/install.conf | 74 .../installation/install.sh | 116 - .../package/calico/calico-node.service | 50 --- .../installation/package/calico/calicoctl.cfg | 22 - .../installation/package/docker/daemon.json | 23 - .../installation/package/docker/docker.service | 35 -- .../installation/package/etcd/etcd.service | 40 -- .../package/hadoop/container-executor.cfg | 41 -- .../installation/package/submarine/submarine.sh | 25 -- .../installation/scripts/calico.sh | 224 -- .../installation/scripts/docker.sh | 166 --- .../installation/scripts/download-server.sh | 42 -- .../installation/scripts/environment.sh | 213 - .../installation/scripts/etcd.sh| 152 --- .../installation/scripts/hadoop.sh | 117 - .../installation/scripts/menu.sh| 444 --- .../installation/scripts/nvidia-docker.sh | 99 - .../installation/scripts/nvidia.sh | 120 - .../installation/scripts/submarine.sh | 38 -- .../installation/scripts/utils.sh | 123 - 21 files changed, 2172 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/829b530a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 9c401e8..a2ea08c 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -66,14 +66,6 @@ 0755 - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation - /share/hadoop/yarn/submarine-installer - -**/* - - 0755 - - hadoop-yarn/conf etc/hadoop http://git-wip-us.apache.org/repos/asf/hadoop/blob/829b530a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf deleted file mode 100644 index 82dcf61..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# install config # - -# DNS -LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf -YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address - -# etcd hosts list -ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) - -# docker registry ip:port -DOCKER_REGISTRY="10.120.196.232:5000" - -# Start the http download service on the specified server, -# Will download all the dependencies in the http server, -# Run the install script on other servers. -# Automatically download dependencies from http, -# Solve the problem that all servers are slow to
[1/2] hadoop git commit: Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
Repository: hadoop Updated Branches: refs/heads/branch-3.2.0 542b224d1 -> 829b530a0 http://git-wip-us.apache.org/repos/asf/hadoop/blob/829b530a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh deleted file mode 100644 index 7b3c2a9..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description check install user -## @audience public -## @stabilitystable -function check_install_user() -{ - if [[ $(id -u) -ne 0 ]];then -echo "This script must be run with a ROOT user!" -exit # don't call exit_install() - fi -} - -## @description exit install -## @audience public -## @stabilitystable -function exit_install() -{ - echo "Exit the installation!" | tee -a $LOG - exit $1 -} - -## @description Check if the IP address format is correct -## @audience public -## @stabilitystable -function valid_ip() -{ - local ip=$1 - local stat=1 - - if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then -OIFS=$IFS -IFS='.' -ip=($ip) -IFS=$OIFS - -if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]; then - stat=$? -fi - fi - - return $stat -} - -## @description Check if the configuration file configuration is correct -## @audience public -## @stabilitystable -function check_install_conf() -{ - echo "Check if the configuration file configuration is correct ..." | tee -a $LOG - - # check etcd conf - hostCount=${#ETCD_HOSTS[@]} - if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2 -echo "Number of nodes = [$hostCount], must be configured to be greater than or equal to 3 servers! " | tee -a $LOG -exit_install - fi - for ip in ${ETCD_HOSTS[@]} - do -if ! valid_ip $ip; then - echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG - exit_install -fi - done - echo "Check if the configuration file configuration is correct [ Done ]" | tee -a $LOG -} - -## @description index by EtcdHosts list -## @audience public -## @stabilitystable -function indexByEtcdHosts() { - index=0 - while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do -if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then - echo $index - return -fi -let "index++" - done - echo "" -} - -## @description get local IP -## @audience public -## @stabilitystable -function getLocalIP() -{ - local _ip _myip _line _nl=$'\n' - while IFS=$': \t' read -a _line ;do - [ -z "${_line%inet}" ] && - _ip=${_line[${#_line[1]}>4?1:2]} && - [ "${_ip#127.0.0.1}" ] && _myip=$_ip -done< <(LANG=C /sbin/ifconfig) - printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip -} - -## @description get ip list -## @audience public -## @stabilitystable -function get_ip_list() -{ - array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[ \t]*//g' | cut -d ' ' -f2) - - for ip in ${array[@]} - do -LOCAL_HOST_IP_LIST+=(${ip}) - done -} - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)" This reverts commit 920401b2966c2dcd7cd1b3dfd7a764f08412323f. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/559cb99b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/559cb99b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/559cb99b Branch: refs/heads/branch-3.2 Commit: 559cb99bf69f9af4b101bd3f0c1d6d5f99f02a6f Parents: 05e41dd Author: Wangda Tan Authored: Tue Dec 4 14:13:39 2018 -0800 Committer: Wangda Tan Committed: Tue Dec 4 14:13:39 2018 -0800 -- .../resources/assemblies/hadoop-yarn-dist.xml | 8 - .../installation/install.conf | 74 .../installation/install.sh | 116 - .../package/calico/calico-node.service | 50 --- .../installation/package/calico/calicoctl.cfg | 22 - .../installation/package/docker/daemon.json | 23 - .../installation/package/docker/docker.service | 35 -- .../installation/package/etcd/etcd.service | 40 -- .../package/hadoop/container-executor.cfg | 41 -- .../installation/package/submarine/submarine.sh | 25 -- .../installation/scripts/calico.sh | 224 -- .../installation/scripts/docker.sh | 166 --- .../installation/scripts/download-server.sh | 42 -- .../installation/scripts/environment.sh | 213 - .../installation/scripts/etcd.sh| 152 --- .../installation/scripts/hadoop.sh | 117 - .../installation/scripts/menu.sh| 444 --- .../installation/scripts/nvidia-docker.sh | 99 - .../installation/scripts/nvidia.sh | 120 - .../installation/scripts/submarine.sh | 38 -- .../installation/scripts/utils.sh | 123 - 21 files changed, 2172 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/559cb99b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 9c401e8..a2ea08c 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -66,14 +66,6 @@ 0755 - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation - /share/hadoop/yarn/submarine-installer - -**/* - - 0755 - - hadoop-yarn/conf etc/hadoop http://git-wip-us.apache.org/repos/asf/hadoop/blob/559cb99b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf deleted file mode 100644 index 82dcf61..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# install config # - -# DNS -LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf -YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address - -# etcd hosts list -ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) - -# docker registry ip:port -DOCKER_REGISTRY="10.120.196.232:5000" - -# Start the http download service on the specified server, -# Will download all the dependencies in the http server, -# Run the install script on other servers. -# Automatically download dependencies from http, -# Solve the problem that all servers are slow to
[1/2] hadoop git commit: Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
Repository: hadoop Updated Branches: refs/heads/branch-3.2 05e41dd4f -> 559cb99bf http://git-wip-us.apache.org/repos/asf/hadoop/blob/559cb99b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh deleted file mode 100644 index 7b3c2a9..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description check install user -## @audience public -## @stabilitystable -function check_install_user() -{ - if [[ $(id -u) -ne 0 ]];then -echo "This script must be run with a ROOT user!" -exit # don't call exit_install() - fi -} - -## @description exit install -## @audience public -## @stabilitystable -function exit_install() -{ - echo "Exit the installation!" | tee -a $LOG - exit $1 -} - -## @description Check if the IP address format is correct -## @audience public -## @stabilitystable -function valid_ip() -{ - local ip=$1 - local stat=1 - - if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then -OIFS=$IFS -IFS='.' -ip=($ip) -IFS=$OIFS - -if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]; then - stat=$? -fi - fi - - return $stat -} - -## @description Check if the configuration file configuration is correct -## @audience public -## @stabilitystable -function check_install_conf() -{ - echo "Check if the configuration file configuration is correct ..." | tee -a $LOG - - # check etcd conf - hostCount=${#ETCD_HOSTS[@]} - if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2 -echo "Number of nodes = [$hostCount], must be configured to be greater than or equal to 3 servers! " | tee -a $LOG -exit_install - fi - for ip in ${ETCD_HOSTS[@]} - do -if ! valid_ip $ip; then - echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG - exit_install -fi - done - echo "Check if the configuration file configuration is correct [ Done ]" | tee -a $LOG -} - -## @description index by EtcdHosts list -## @audience public -## @stabilitystable -function indexByEtcdHosts() { - index=0 - while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do -if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then - echo $index - return -fi -let "index++" - done - echo "" -} - -## @description get local IP -## @audience public -## @stabilitystable -function getLocalIP() -{ - local _ip _myip _line _nl=$'\n' - while IFS=$': \t' read -a _line ;do - [ -z "${_line%inet}" ] && - _ip=${_line[${#_line[1]}>4?1:2]} && - [ "${_ip#127.0.0.1}" ] && _myip=$_ip -done< <(LANG=C /sbin/ifconfig) - printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip -} - -## @description get ip list -## @audience public -## @stabilitystable -function get_ip_list() -{ - array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[ \t]*//g' | cut -d ' ' -f2) - - for ip in ${array[@]} - do -LOCAL_HOST_IP_LIST+=(${ip}) - done -} - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)" This reverts commit 46d6e0016610ced51a76189daeb3ad0e3dbbf94c. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/228156cf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/228156cf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/228156cf Branch: refs/heads/trunk Commit: 228156cfd1b474988bc4fedfbf7edddc87db41e3 Parents: 9287ab3 Author: Wangda Tan Authored: Tue Dec 4 14:13:06 2018 -0800 Committer: Wangda Tan Committed: Tue Dec 4 14:13:06 2018 -0800 -- .../resources/assemblies/hadoop-yarn-dist.xml | 8 - .../installation/install.conf | 74 .../installation/install.sh | 116 - .../package/calico/calico-node.service | 50 --- .../installation/package/calico/calicoctl.cfg | 22 - .../installation/package/docker/daemon.json | 23 - .../installation/package/docker/docker.service | 35 -- .../installation/package/etcd/etcd.service | 40 -- .../package/hadoop/container-executor.cfg | 41 -- .../installation/package/submarine/submarine.sh | 25 -- .../installation/scripts/calico.sh | 224 -- .../installation/scripts/docker.sh | 166 --- .../installation/scripts/download-server.sh | 42 -- .../installation/scripts/environment.sh | 213 - .../installation/scripts/etcd.sh| 152 --- .../installation/scripts/hadoop.sh | 117 - .../installation/scripts/menu.sh| 444 --- .../installation/scripts/nvidia-docker.sh | 99 - .../installation/scripts/nvidia.sh | 120 - .../installation/scripts/submarine.sh | 38 -- .../installation/scripts/utils.sh | 123 - 21 files changed, 2172 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 69ab66a..4055acb 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -66,14 +66,6 @@ 0755 - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation - /share/hadoop/yarn/submarine-installer - -**/* - - 0755 - - hadoop-yarn/conf etc/hadoop http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf deleted file mode 100644 index 82dcf61..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# install config # - -# DNS -LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf -YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address - -# etcd hosts list -ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) - -# docker registry ip:port -DOCKER_REGISTRY="10.120.196.232:5000" - -# Start the http download service on the specified server, -# Will download all the dependencies in the http server, -# Run the install script on other servers. -# Automatically download dependencies from http, -# Solve the problem that all servers are slow to
[1/2] hadoop git commit: Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
Repository: hadoop Updated Branches: refs/heads/trunk 9287ab364 -> 228156cfd http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh deleted file mode 100644 index 7b3c2a9..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description check install user -## @audience public -## @stabilitystable -function check_install_user() -{ - if [[ $(id -u) -ne 0 ]];then -echo "This script must be run with a ROOT user!" -exit # don't call exit_install() - fi -} - -## @description exit install -## @audience public -## @stabilitystable -function exit_install() -{ - echo "Exit the installation!" | tee -a $LOG - exit $1 -} - -## @description Check if the IP address format is correct -## @audience public -## @stabilitystable -function valid_ip() -{ - local ip=$1 - local stat=1 - - if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then -OIFS=$IFS -IFS='.' -ip=($ip) -IFS=$OIFS - -if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]; then - stat=$? -fi - fi - - return $stat -} - -## @description Check if the configuration file configuration is correct -## @audience public -## @stabilitystable -function check_install_conf() -{ - echo "Check if the configuration file configuration is correct ..." | tee -a $LOG - - # check etcd conf - hostCount=${#ETCD_HOSTS[@]} - if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2 -echo "Number of nodes = [$hostCount], must be configured to be greater than or equal to 3 servers! " | tee -a $LOG -exit_install - fi - for ip in ${ETCD_HOSTS[@]} - do -if ! valid_ip $ip; then - echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG - exit_install -fi - done - echo "Check if the configuration file configuration is correct [ Done ]" | tee -a $LOG -} - -## @description index by EtcdHosts list -## @audience public -## @stabilitystable -function indexByEtcdHosts() { - index=0 - while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do -if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then - echo $index - return -fi -let "index++" - done - echo "" -} - -## @description get local IP -## @audience public -## @stabilitystable -function getLocalIP() -{ - local _ip _myip _line _nl=$'\n' - while IFS=$': \t' read -a _line ;do - [ -z "${_line%inet}" ] && - _ip=${_line[${#_line[1]}>4?1:2]} && - [ "${_ip#127.0.0.1}" ] && _myip=$_ip -done< <(LANG=C /sbin/ifconfig) - printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip -} - -## @description get ip list -## @audience public -## @stabilitystable -function get_ip_list() -{ - array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[ \t]*//g' | cut -d ' ' -f2) - - for ip in ${array[@]} - do -LOCAL_HOST_IP_LIST+=(${ip}) - done -} - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-9010. Fix the incorrect trailing slash deletion in constructor method of CGroupsHandlerImpl. (Zhankun Tang via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 0081b02e3 -> bad12031f YARN-9010. Fix the incorrect trailing slash deletion in constructor method of CGroupsHandlerImpl. (Zhankun Tang via wangda) Change-Id: Iaecc66d57781cc10f19ead4647e47fc9556676da Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bad12031 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bad12031 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bad12031 Branch: refs/heads/trunk Commit: bad12031f603347a701249a1e3ef5d879a5f1c8f Parents: 0081b02 Author: Wangda Tan Authored: Thu Nov 29 14:56:07 2018 -0800 Committer: Wangda Tan Committed: Thu Nov 29 14:56:07 2018 -0800 -- .../linux/resources/CGroupsHandlerImpl.java | 3 +- .../linux/resources/TestCGroupsHandlerImpl.java | 38 2 files changed, 40 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bad12031/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java index 050d0a8..1b2c780 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java @@ -87,9 +87,10 @@ class CGroupsHandlerImpl implements CGroupsHandler { CGroupsHandlerImpl(Configuration conf, PrivilegedOperationExecutor privilegedOperationExecutor, String mtab) throws ResourceHandlerException { +// Remove leading and trialing slash(es) this.cGroupPrefix = conf.get(YarnConfiguration. NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, "/hadoop-yarn") -.replaceAll("^/", "").replaceAll("$/", ""); +.replaceAll("^/+", "").replaceAll("/+$", ""); this.enableCGroupMount = conf.getBoolean(YarnConfiguration. NM_LINUX_CONTAINER_CGROUPS_MOUNT, false); this.cGroupMountPath = conf.get(YarnConfiguration. http://git-wip-us.apache.org/repos/asf/hadoop/blob/bad12031/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java index ea6fb52..70badaf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java @@ -598,4 +598,42 @@ public class TestCGroupsHandlerImpl { FileUtils.deleteQuietly(cpu); } } + + // Remove leading and trailing slashes + @Test + public void testCgroupsHierarchySetting() throws ResourceHandlerException { +YarnConfiguration conf = new YarnConfiguration(); +conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, tmpPath); +conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, +"/hadoop-yarn"); +CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(conf, null); +String expectedRelativePath = "hadoop-yarn/c1"; +Assert.assertEquals(expectedRelativePath, +cGroupsHandler.getRelativePathForCGroup("c1")); + +conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGR
hadoop git commit: YARN-9030. Log aggregation changes to handle filesystems which do not support setting permissions. (Suma Shivaprasad via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.2 7a78bdf7b -> ee9deb6e9 YARN-9030. Log aggregation changes to handle filesystems which do not support setting permissions. (Suma Shivaprasad via wangda) Change-Id: I80f1e8196b8624e24d74494719fdedfd7061dced (cherry picked from commit 9de8e8d0496a2628b63cc841b1fdee80e2912f7a) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee9deb6e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee9deb6e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee9deb6e Branch: refs/heads/branch-3.2 Commit: ee9deb6e9ccebcf6ef2660e32b0d26f458b3ee5d Parents: 7a78bdf Author: Wangda Tan Authored: Wed Nov 21 17:28:37 2018 -0800 Committer: Wangda Tan Committed: Wed Nov 28 15:06:35 2018 -0800 -- .../LogAggregationFileController.java | 74 +++- 1 file changed, 56 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee9deb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index fe65288..e37308d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -109,6 +109,8 @@ public abstract class LogAggregationFileController { protected int retentionSize; protected String fileControllerName; + protected boolean fsSupportsChmod = true; + public LogAggregationFileController() {} /** @@ -250,7 +252,6 @@ public abstract class LogAggregationFileController { * Verify and create the remote log directory. */ public void verifyAndCreateRemoteLogDir() { -boolean logPermError = true; // Checking the existence of the TLD FileSystem remoteFS = null; try { @@ -264,14 +265,12 @@ public abstract class LogAggregationFileController { try { FsPermission perms = remoteFS.getFileStatus(remoteRootLogDir).getPermission(); - if (!perms.equals(TLDIR_PERMISSIONS) && logPermError) { + if (!perms.equals(TLDIR_PERMISSIONS)) { LOG.warn("Remote Root Log Dir [" + remoteRootLogDir + "] already exist, but with incorrect permissions. " + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms + "]." + " The cluster may have problems with multiple users."); -logPermError = false; - } else { -logPermError = true; + } } catch (FileNotFoundException e) { remoteExists = false; @@ -280,15 +279,26 @@ public abstract class LogAggregationFileController { "Failed to check permissions for dir [" + remoteRootLogDir + "]", e); } + +Path qualified = +remoteRootLogDir.makeQualified(remoteFS.getUri(), +remoteFS.getWorkingDirectory()); if (!remoteExists) { LOG.warn("Remote Root Log Dir [" + remoteRootLogDir + "] does not exist. Attempting to create it."); try { -Path qualified = -remoteRootLogDir.makeQualified(remoteFS.getUri(), -remoteFS.getWorkingDirectory()); remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS)); -remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS)); + +// Not possible to query FileSystem API to check if it supports +// chmod, chown etc. Hence resorting to catching exceptions here. +// Remove when FS APi is ready +try { + remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS)); +} catch ( UnsupportedOperationException use) { + LOG.info("Unable to set permissions for configured filesystem since" + + " it does not support this", remoteFS.getScheme()); + fsSupportsChmod = false; +} UserGroupInformation loginUser = UserGroupInformation.getLoginUser(); String primaryGroupName = null; @@ -301,13 +311,31 @@ public abstract class LogAggregationFileController { } // set owner on th
[4/6] hadoop git commit: YARN-8974. Improve the assertion message in TestGPUResourceHandler. (Zhankun Tang via wangda)
YARN-8974. Improve the assertion message in TestGPUResourceHandler. (Zhankun Tang via wangda) Change-Id: I4eb58e9d251d5f54e7feffc4fbb813b4f5ae4b1b Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ebeda98 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ebeda98 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ebeda98 Branch: refs/heads/trunk Commit: 8ebeda98a9d3eac45598a33bae3d62e3ebb92cad Parents: 9ed8756 Author: Wangda Tan Authored: Wed Nov 28 14:36:30 2018 -0800 Committer: Wangda Tan Committed: Wed Nov 28 14:36:30 2018 -0800 -- .../linux/resources/gpu/TestGpuResourceHandler.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ebeda98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java index 10e5cd1..18785e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java @@ -465,7 +465,7 @@ public class TestGpuResourceHandler { caughtException = true; } Assert.assertTrue( -"Should fail since requested device Id is not in allowed list", +"Should fail since requested device Id is already assigned", caughtException); // Make sure internal state not changed. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[3/6] hadoop git commit: YARN-9061. Improve the GPU/FPGA module log message of container-executor. (Zhankun Tang via wangda)
YARN-9061. Improve the GPU/FPGA module log message of container-executor. (Zhankun Tang via wangda) Change-Id: Iece9b47438357077a53984a820d4d6423f480518 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ed87567 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ed87567 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ed87567 Branch: refs/heads/trunk Commit: 9ed87567ad0f1c26a263ce6d8fba56d066260c5c Parents: 579ef4b Author: Wangda Tan Authored: Wed Nov 28 14:31:31 2018 -0800 Committer: Wangda Tan Committed: Wed Nov 28 14:31:31 2018 -0800 -- .../native/container-executor/impl/modules/fpga/fpga-module.c | 5 +++-- .../native/container-executor/impl/modules/gpu/gpu-module.c | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed87567/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c index c1a2f83..e947d7c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c @@ -141,7 +141,7 @@ void reload_fpga_configuration() { /* * Format of FPGA request commandline: * - * c-e fpga --excluded_fpgas 0,1,3 --container_id container_x_y + * c-e --module-fpga --excluded_fpgas 0,1,3 --container_id container_x_y */ int handle_fpga_request(update_cgroups_parameters_function func, const char* module_name, int module_argc, char** module_argv) { @@ -213,7 +213,8 @@ int handle_fpga_request(update_cgroups_parameters_function func, if (!minor_devices) { // Minor devices is null, skip following call. - fprintf(ERRORFILE, "is not specified, skip cgroups call.\n"); + fprintf(ERRORFILE, + "--excluded-fpgas is not specified, skip cgroups call.\n"); goto cleanup; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed87567/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c index 1a1b164..7522338 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c @@ -141,7 +141,7 @@ void reload_gpu_configuration() { /* * Format of GPU request commandline: * - * c-e gpu --excluded_gpus 0,1,3 --container_id container_x_y + * c-e --module-gpu --excluded_gpus 0,1,3 --container_id container_x_y */ int handle_gpu_request(update_cgroups_parameters_func func, const char* module_name, int module_argc, char** module_argv) { @@ -213,7 +213,8 @@ int handle_gpu_request(update_cgroups_parameters_func func, if (!minor_devices) { // Minor devices is null, skip following call. - fprintf(ERRORFILE, "is not specified, skip cgroups call.\n"); + fprintf(ERRORFILE, + "--excluded_gpus is not specified, skip cgroups call.\n"); goto cleanup; } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/6] hadoop git commit: YARN-9030. Log aggregation changes to handle filesystems which do not support setting permissions. (Suma Shivaprasad via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 4d8de7ab6 -> fe7dab8ef YARN-9030. Log aggregation changes to handle filesystems which do not support setting permissions. (Suma Shivaprasad via wangda) Change-Id: I80f1e8196b8624e24d74494719fdedfd7061dced Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9de8e8d0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9de8e8d0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9de8e8d0 Branch: refs/heads/trunk Commit: 9de8e8d0496a2628b63cc841b1fdee80e2912f7a Parents: 4d8de7a Author: Wangda Tan Authored: Wed Nov 21 17:28:37 2018 -0800 Committer: Wangda Tan Committed: Wed Nov 28 13:36:21 2018 -0800 -- .../LogAggregationFileController.java | 74 +++- 1 file changed, 56 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9de8e8d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index fe65288..e37308d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -109,6 +109,8 @@ public abstract class LogAggregationFileController { protected int retentionSize; protected String fileControllerName; + protected boolean fsSupportsChmod = true; + public LogAggregationFileController() {} /** @@ -250,7 +252,6 @@ public abstract class LogAggregationFileController { * Verify and create the remote log directory. */ public void verifyAndCreateRemoteLogDir() { -boolean logPermError = true; // Checking the existence of the TLD FileSystem remoteFS = null; try { @@ -264,14 +265,12 @@ public abstract class LogAggregationFileController { try { FsPermission perms = remoteFS.getFileStatus(remoteRootLogDir).getPermission(); - if (!perms.equals(TLDIR_PERMISSIONS) && logPermError) { + if (!perms.equals(TLDIR_PERMISSIONS)) { LOG.warn("Remote Root Log Dir [" + remoteRootLogDir + "] already exist, but with incorrect permissions. " + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms + "]." + " The cluster may have problems with multiple users."); -logPermError = false; - } else { -logPermError = true; + } } catch (FileNotFoundException e) { remoteExists = false; @@ -280,15 +279,26 @@ public abstract class LogAggregationFileController { "Failed to check permissions for dir [" + remoteRootLogDir + "]", e); } + +Path qualified = +remoteRootLogDir.makeQualified(remoteFS.getUri(), +remoteFS.getWorkingDirectory()); if (!remoteExists) { LOG.warn("Remote Root Log Dir [" + remoteRootLogDir + "] does not exist. Attempting to create it."); try { -Path qualified = -remoteRootLogDir.makeQualified(remoteFS.getUri(), -remoteFS.getWorkingDirectory()); remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS)); -remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS)); + +// Not possible to query FileSystem API to check if it supports +// chmod, chown etc. Hence resorting to catching exceptions here. +// Remove when FS APi is ready +try { + remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS)); +} catch ( UnsupportedOperationException use) { + LOG.info("Unable to set permissions for configured filesystem since" + + " it does not support this", remoteFS.getScheme()); + fsSupportsChmod = false; +} UserGroupInformation loginUser = UserGroupInformation.getLoginUser(); String primaryGroupName = null; @@ -301,13 +311,31 @@ public abstract class LogAggregationFileController { } // set owner on the remote directory only if the primary group exists if (primaryGroupNam
[6/6] hadoop git commit: YARN-8989. [YARN-8851] Move DockerCommandPlugin volume related APIs' invocation from DockerLinuxContainerRuntime#prepareContainer to #launchContainer. (Zhankun Tang via wangda
YARN-8989. [YARN-8851] Move DockerCommandPlugin volume related APIs' invocation from DockerLinuxContainerRuntime#prepareContainer to #launchContainer. (Zhankun Tang via wangda) Change-Id: Ia6d532c687168448416dfdf46f0ac34bff20e6ca Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe7dab8e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe7dab8e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe7dab8e Branch: refs/heads/trunk Commit: fe7dab8ef55f08cf18c2d62c782c1ab8930a5a15 Parents: 8976439 Author: Wangda Tan Authored: Wed Nov 28 14:55:16 2018 -0800 Committer: Wangda Tan Committed: Wed Nov 28 15:03:06 2018 -0800 -- .../runtime/DockerLinuxContainerRuntime.java| 44 .../runtime/TestDockerContainerRuntime.java | 15 --- 2 files changed, 24 insertions(+), 35 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe7dab8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 15ff0ff..225bc19 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -456,32 +456,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { @Override public void prepareContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { -Container container = ctx.getContainer(); - -// Create volumes when needed. -if (nmContext != null -&& nmContext.getResourcePluginManager().getNameToPlugins() != null) { - for (ResourcePlugin plugin : nmContext.getResourcePluginManager() - .getNameToPlugins().values()) { -DockerCommandPlugin dockerCommandPlugin = -plugin.getDockerCommandPluginInstance(); -if (dockerCommandPlugin != null) { - DockerVolumeCommand dockerVolumeCommand = - dockerCommandPlugin.getCreateDockerVolumeCommand( - ctx.getContainer()); - if (dockerVolumeCommand != null) { -runDockerVolumeCommand(dockerVolumeCommand, container); - -// After volume created, run inspect to make sure volume properly -// created. -if (dockerVolumeCommand.getSubCommand().equals( -DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND)) { - checkDockerVolumeCreated(dockerVolumeCommand, container); -} - } -} - } -} } private void checkDockerVolumeCreated( @@ -1034,14 +1008,30 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { } } -// use plugins to update docker run command. +// use plugins to create volume and update docker run command. if (nmContext != null && nmContext.getResourcePluginManager().getNameToPlugins() != null) { for (ResourcePlugin plugin : nmContext.getResourcePluginManager() .getNameToPlugins().values()) { DockerCommandPlugin dockerCommandPlugin = plugin.getDockerCommandPluginInstance(); + if (dockerCommandPlugin != null) { + // Create volumes when needed. + DockerVolumeCommand dockerVolumeCommand = + dockerCommandPlugin.getCreateDockerVolumeCommand( + ctx.getContainer()); + if (dockerVolumeCommand != null) { +runDockerVolumeCommand(dockerVolumeCommand, container); + +// After volume created, run inspect to make sure volume properly +// created. +if (dockerVolumeCommand.getSubCommand().equals( +DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND)) { + checkDockerVolumeCreated(dockerVolumeCommand, container); +} + } + // Update cmd dockerCommandPlugin.updateDockerRunCommand(runCommand, container);
[2/6] hadoop git commit: YARN-8882. [YARN-8851] Add a shared device mapping manager (scheduler) for device plugins. (Zhankun Tang via wangda)
YARN-8882. [YARN-8851] Add a shared device mapping manager (scheduler) for device plugins. (Zhankun Tang via wangda) Change-Id: I9435136642c3d556971a357bf687f69df90bb45e Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/579ef4be Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/579ef4be Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/579ef4be Branch: refs/heads/trunk Commit: 579ef4be063745c5211127eca83a393ceddc8b79 Parents: 9de8e8d Author: Wangda Tan Authored: Wed Nov 28 14:09:52 2018 -0800 Committer: Wangda Tan Committed: Wed Nov 28 14:09:52 2018 -0800 -- .../resourceplugin/ResourcePluginManager.java | 14 +- .../deviceframework/DeviceMappingManager.java | 324 .../deviceframework/DevicePluginAdapter.java| 20 +- .../DeviceResourceHandlerImpl.java | 145 +++ .../TestDeviceMappingManager.java | 366 + .../TestDevicePluginAdapter.java| 388 ++- 6 files changed, 1245 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/579ef4be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java index 9741b12..6dfe817 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin; import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRegisterRequest; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.DeviceMappingManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.DevicePluginAdapter; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaResourcePlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin; @@ -52,12 +53,13 @@ import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI; public class ResourcePluginManager { private static final Logger LOG = LoggerFactory.getLogger(ResourcePluginManager.class); - private static final Set SUPPORTED_RESOURCE_PLUGINS = ImmutableSet.of( - GPU_URI, FPGA_URI); + private static final Set SUPPORTED_RESOURCE_PLUGINS = + ImmutableSet.of(GPU_URI, FPGA_URI); private Map configuredPlugins = Collections.emptyMap(); + private DeviceMappingManager deviceMappingManager = null; public synchronized void initialize(Context context) throws YarnException, ClassNotFoundException { @@ -123,7 +125,7 @@ public class ResourcePluginManager { throws YarnRuntimeException, ClassNotFoundException { LOG.info("The pluggable device framework enabled," + "trying to load the vendor plugins"); - +deviceMappingManager = new DeviceMappingManager(context); String[] pluginClassNames = configuration.getStrings( YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES); if (null == pluginClassNames) { @@ -174,7 +176,7 @@ public class ResourcePluginManager { resourceName, pluginClassName); DevicePluginAdapter pluginAdapter = new DevicePluginAdapter( - resourceName, dpInstance); + resourceName, dpInstance, deviceMappingManager); LOG.info("Adapter of {} created. Initializing..", pluginClassName); try { pluginAdapter.initialize(context); @@ -235,6 +237,10 @@ public class ResourcePluginManager { return true; } + public DeviceMappingManager getDeviceMappingManager() { +return deviceMappingManager; + } + public synchronized void
[5/6] hadoop git commit: YARN-8975. [Submarine] Use predefined Charset object StandardCharsets.UTF_8 instead of String UTF-8. (Zhankun Tang via wangda)
YARN-8975. [Submarine] Use predefined Charset object StandardCharsets.UTF_8 instead of String UTF-8. (Zhankun Tang via wangda) Change-Id: If6c7904aa17895e543cfca245264249eb7328bdc Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89764392 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89764392 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89764392 Branch: refs/heads/trunk Commit: 897643928c534062d45d00a36b95fd99b4f6 Parents: 8ebeda9 Author: Wangda Tan Authored: Wed Nov 28 14:39:06 2018 -0800 Committer: Wangda Tan Committed: Wed Nov 28 14:39:06 2018 -0800 -- .../submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/89764392/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java index b58ad77..2e84c96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java @@ -49,6 +49,7 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.Writer; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -218,7 +219,8 @@ public class YarnServiceJobSubmitter implements JobSubmitter { private String generateCommandLaunchScript(RunJobParameters parameters, TaskType taskType, Component comp) throws IOException { File file = File.createTempFile(taskType.name() + "-launch-script", ".sh"); -Writer w = new OutputStreamWriter(new FileOutputStream(file), "UTF-8"); +Writer w = new OutputStreamWriter(new FileOutputStream(file), +StandardCharsets.UTF_8); PrintWriter pw = new PrintWriter(w); try { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-8960. [Submarine] Can't get submarine service status using the command of "yarn app -status" under security environment. (Zac Zhou via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.2 0b2cfc8ab -> 34387599c YARN-8960. [Submarine] Can't get submarine service status using the command of "yarn app -status" under security environment. (Zac Zhou via wangda) Change-Id: I21b1addc9c32817650ea744a8f2e6b5602f2f4d4 (cherry picked from commit 8b2381441558cd49b4c940b0760c8accbb2a5567) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34387599 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34387599 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34387599 Branch: refs/heads/branch-3.2 Commit: 34387599c41b5026a7ab179b31fbc09cfcbd5071 Parents: 0b2cfc8 Author: Wangda Tan Authored: Mon Nov 19 08:55:30 2018 -0800 Committer: Wangda Tan Committed: Mon Nov 19 09:01:21 2018 -0800 -- .../hadoop/yarn/submarine/client/cli/Cli.java | 3 - .../yarn/submarine/client/cli/CliConstants.java | 3 + .../yarn/submarine/client/cli/CliUtils.java | 56 ++ .../yarn/submarine/client/cli/RunJobCli.java| 12 +++- .../client/cli/param/RunJobParameters.java | 46 ++- .../yarn/submarine/common/ClientContext.java| 15 +++-- .../fs/DefaultRemoteDirectoryManager.java | 3 +- .../yarnservice/YarnServiceJobSubmitter.java| 61 +--- .../client/cli/TestRunJobCliParsing.java| 9 ++- 9 files changed, 188 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/34387599/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java index b4c5e4c..69189f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java @@ -16,7 +16,6 @@ package org.apache.hadoop.yarn.submarine.client.cli; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.submarine.common.ClientContext; -import org.apache.hadoop.yarn.submarine.common.fs.DefaultRemoteDirectoryManager; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.submarine.runtimes.RuntimeFactory; import org.slf4j.Logger; @@ -43,8 +42,6 @@ public class Cli { Configuration conf = new YarnConfiguration(); ClientContext clientContext = new ClientContext(); clientContext.setConfiguration(conf); -clientContext.setRemoteDirectoryManager( -new DefaultRemoteDirectoryManager(clientContext)); RuntimeFactory runtimeFactory = RuntimeFactory.getRuntimeFactory( clientContext); clientContext.setRuntimeFactory(runtimeFactory); http://git-wip-us.apache.org/repos/asf/hadoop/blob/34387599/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java index 454ff1c..2d7a472 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java @@ -52,4 +52,7 @@ public class CliConstants { public static final String QUICKLINK = "quicklink"; public static final String TENSORBOARD_DOCKER_IMAGE = "tensorboard_docker_image"; + public static final String KEYTAB = "keytab"; + public static final String PRINCIPAL = "principal"; + public static final String DISTRIBUTE_KEYTAB = "distribute_keytab"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/34387599/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src
[2/2] hadoop git commit: YARN-8960. [Submarine] Can't get submarine service status using the command of "yarn app -status" under security environment. (Zac Zhou via wangda)
YARN-8960. [Submarine] Can't get submarine service status using the command of "yarn app -status" under security environment. (Zac Zhou via wangda) Change-Id: I21b1addc9c32817650ea744a8f2e6b5602f2f4d4 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b238144 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b238144 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b238144 Branch: refs/heads/trunk Commit: 8b2381441558cd49b4c940b0760c8accbb2a5567 Parents: 6357803 Author: Wangda Tan Authored: Mon Nov 19 08:55:30 2018 -0800 Committer: Wangda Tan Committed: Mon Nov 19 08:55:30 2018 -0800 -- .../hadoop/yarn/submarine/client/cli/Cli.java | 3 - .../yarn/submarine/client/cli/CliConstants.java | 3 + .../yarn/submarine/client/cli/CliUtils.java | 56 ++ .../yarn/submarine/client/cli/RunJobCli.java| 12 +++- .../client/cli/param/RunJobParameters.java | 46 ++- .../yarn/submarine/common/ClientContext.java| 15 +++-- .../fs/DefaultRemoteDirectoryManager.java | 3 +- .../yarnservice/YarnServiceJobSubmitter.java| 61 +--- .../client/cli/TestRunJobCliParsing.java| 9 ++- 9 files changed, 188 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b238144/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java index b4c5e4c..69189f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java @@ -16,7 +16,6 @@ package org.apache.hadoop.yarn.submarine.client.cli; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.submarine.common.ClientContext; -import org.apache.hadoop.yarn.submarine.common.fs.DefaultRemoteDirectoryManager; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.submarine.runtimes.RuntimeFactory; import org.slf4j.Logger; @@ -43,8 +42,6 @@ public class Cli { Configuration conf = new YarnConfiguration(); ClientContext clientContext = new ClientContext(); clientContext.setConfiguration(conf); -clientContext.setRemoteDirectoryManager( -new DefaultRemoteDirectoryManager(clientContext)); RuntimeFactory runtimeFactory = RuntimeFactory.getRuntimeFactory( clientContext); clientContext.setRuntimeFactory(runtimeFactory); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b238144/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java index 454ff1c..2d7a472 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java @@ -52,4 +52,7 @@ public class CliConstants { public static final String QUICKLINK = "quicklink"; public static final String TENSORBOARD_DOCKER_IMAGE = "tensorboard_docker_image"; + public static final String KEYTAB = "keytab"; + public static final String PRINCIPAL = "principal"; + public static final String DISTRIBUTE_KEYTAB = "distribute_keytab"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b238144/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-su
[1/2] hadoop git commit: YARN-8881. [YARN-8851] Add basic pluggable device plugin framework. (Zhankun Tang via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 93666087b -> 8b2381441 YARN-8881. [YARN-8851] Add basic pluggable device plugin framework. (Zhankun Tang via wangda) Change-Id: If9a2f68cd4713b4ec932cdeda68106f17437c3d3 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/63578036 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/63578036 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/63578036 Branch: refs/heads/trunk Commit: 63578036450f660d49ae204327efcd629d9dd137 Parents: 9366608 Author: Wangda Tan Authored: Mon Nov 19 08:54:31 2018 -0800 Committer: Wangda Tan Committed: Mon Nov 19 08:54:31 2018 -0800 -- .../nodemanager/api/deviceplugin/Device.java| 233 +++ .../api/deviceplugin/DevicePlugin.java | 65 ++ .../api/deviceplugin/DeviceRegisterRequest.java | 73 ++ .../api/deviceplugin/DeviceRuntimeSpec.java | 137 +++ .../api/deviceplugin/MountDeviceSpec.java | 131 +++ .../api/deviceplugin/MountVolumeSpec.java | 118 ++ .../api/deviceplugin/VolumeSpec.java| 114 + .../api/deviceplugin/YarnRuntimeType.java | 42 .../api/deviceplugin/package-info.java | 19 ++ .../resourceplugin/ResourcePluginManager.java | 116 - .../deviceframework/DevicePluginAdapter.java| 88 +++ .../DeviceResourceUpdaterImpl.java | 66 ++ .../deviceframework/package-info.java | 19 ++ .../resourceplugin/package-info.java| 19 ++ .../TestResourcePluginManager.java | 169 +- .../deviceframework/FakeTestDevicePlugin1.java | 61 + .../deviceframework/FakeTestDevicePlugin2.java | 27 +++ .../deviceframework/FakeTestDevicePlugin3.java | 65 ++ .../deviceframework/FakeTestDevicePlugin4.java | 54 + .../TestDevicePluginAdapter.java| 145 .../resource-types-pluggable-devices.xml| 23 ++ 21 files changed, 1773 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/63578036/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/Device.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/Device.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/Device.java new file mode 100644 index 000..551c2d8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/Device.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin; + +import java.io.Serializable; +import java.util.Objects; + +/** + * Represent one "device" resource. + * */ +public final class Device implements Serializable, Comparable { + + private static final long serialVersionUID = -7270474563684671656L; + + /** + * An plugin specified index number. + * Must set. Recommend starting from 0 + * */ + private final int id; + + /** + * The device node like "/dev/devname". + * Optional + * */ + private final String devPath; + + /** + * The major device number. + * Optional + * */ + private final int majorNumber; + + /** + * The minor device number. + * Optional + * */ + private final int minorNumber; + + /** + * PCI Bus ID in format ]:]]:][][.[]]. + * Optional. Can get from "lspci -D" in Linux + * */ + private final String busID; + + /** + * Is healthy or not. + * false by default + * */ + private boolean is
hadoop git commit: YARN-8299. Yarn Service Upgrade: Add GET APIs that returns instances matching query params. (Chandni Singh via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.1 095635d98 -> 077a7ab37 YARN-8299. Yarn Service Upgrade: Add GET APIs that returns instances matching query params. (Chandni Singh via wangda) Change-Id: I7f2aef89c6053d5dc5955f79fdfcd4cd74f74737 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/077a7ab3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/077a7ab3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/077a7ab3 Branch: refs/heads/branch-3.1 Commit: 077a7ab37bd59ae63743dc94020cdaf5c62af396 Parents: 095635d Author: Wangda Tan Authored: Fri Nov 16 10:32:10 2018 -0800 Committer: Wangda Tan Committed: Fri Nov 16 10:32:10 2018 -0800 -- .../yarn/service/client/ApiServiceClient.java | 73 ++--- .../hadoop/yarn/service/webapp/ApiServer.java | 67 ++-- .../hadoop/yarn/service/ClientAMProtocol.java | 5 + .../hadoop/yarn/service/ClientAMService.java| 14 ++ .../yarn/service/client/ServiceClient.java | 47 ++ .../component/instance/ComponentInstance.java | 41 - .../yarn/service/conf/RestApiConstants.java | 5 +- .../pb/client/ClientAMProtocolPBClientImpl.java | 13 ++ .../service/ClientAMProtocolPBServiceImpl.java | 13 ++ .../hadoop/yarn/service/utils/FilterUtils.java | 81 ++ .../yarn/service/utils/ServiceApiUtil.java | 9 ++ .../src/main/proto/ClientAMProtocol.proto | 12 ++ .../yarn/service/MockRunningServiceContext.java | 154 +++ .../yarn/service/client/TestServiceCLI.java | 25 ++- .../yarn/service/client/TestServiceClient.java | 54 ++- .../yarn/service/component/TestComponent.java | 133 +--- .../instance/TestComponentInstance.java | 46 +++--- .../yarn/service/utils/TestFilterUtils.java | 102 .../hadoop/yarn/client/cli/ApplicationCLI.java | 68 +++- .../hadoop/yarn/client/cli/TestYarnCLI.java | 6 +- .../hadoop/yarn/client/api/AppAdminClient.java | 6 + 21 files changed, 772 insertions(+), 202 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/077a7ab3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java index 074b45d..e0b3d86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java @@ -29,6 +29,7 @@ import java.util.Map; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.UriBuilder; import com.google.common.base.Preconditions; import org.apache.commons.codec.binary.Base64; @@ -55,10 +56,8 @@ import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.api.records.ServiceStatus; import org.apache.hadoop.yarn.service.conf.RestApiConstants; -import org.apache.hadoop.yarn.service.utils.JsonSerDeser; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.util.RMHAUtils; -import org.codehaus.jackson.map.PropertyNamingStrategy; import org.eclipse.jetty.util.UrlEncoded; import org.ietf.jgss.GSSContext; import org.ietf.jgss.GSSException; @@ -216,11 +215,7 @@ public class ApiServiceClient extends AppAdminClient { api.append("/"); api.append(appName); } -Configuration conf = getConfig(); -if (conf.get("hadoop.http.authentication.type").equalsIgnoreCase("simple")) { - api.append("?user.name=" + UrlEncoded - .encodeString(System.getProperty("user.name"))); -} +appendUserNameIfRequired(api); return api.toString(); } @@ -231,15 +226,27 @@ public class ApiServiceClient extends AppAdminClient { api.append(url); api.append("/app/v1/services/").append(appName).append("/") .append(RestApiConstants.COMP_INSTANCES); -Configuration conf = getConfig(); -if (conf.get("hadoop.http.a
hadoop git commit: YARN-9001. [Submarine] Use AppAdminClient instead of ServiceClient to sumbit jobs. (Zac Zhou via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.2 f5e1bad0f -> bee5bf867 YARN-9001. [Submarine] Use AppAdminClient instead of ServiceClient to sumbit jobs. (Zac Zhou via wangda) Change-Id: I7e8d1c27ebd37e0907ca570c4f3d56fe7a859635 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bee5bf86 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bee5bf86 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bee5bf86 Branch: refs/heads/branch-3.2 Commit: bee5bf867ba6708f67bb368e2e32c5df538a2e70 Parents: f5e1bad Author: Wangda Tan Authored: Fri Nov 16 10:26:50 2018 -0800 Committer: Wangda Tan Committed: Fri Nov 16 10:26:50 2018 -0800 -- .../yarn/service/client/ServiceClient.java | 1 + .../submarine/runtimes/common/JobMonitor.java | 6 +++ .../yarnservice/YarnServiceJobMonitor.java | 25 +--- .../yarnservice/YarnServiceJobSubmitter.java| 42 ++-- .../runtimes/yarnservice/YarnServiceUtils.java | 15 +++ .../yarnservice/TestYarnServiceRunJobCli.java | 12 +++--- .../submarine/common/MockClientContext.java | 1 - 7 files changed, 79 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bee5bf86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 8724a87..e70e96b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -1360,6 +1360,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, LOG.info("Service {} does not have an application ID", serviceName); return appSpec; } +appSpec.setId(currentAppId.toString()); ApplicationReport appReport = yarnClient.getApplicationReport(currentAppId); appSpec.setState(convertState(appReport.getYarnApplicationState())); ApplicationTimeout lifetime = http://git-wip-us.apache.org/repos/asf/hadoop/blob/bee5bf86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java index c81393b..35e21fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java @@ -48,6 +48,11 @@ public abstract class JobMonitor { throws IOException, YarnException; /** + * Cleanup AppAdminClient, etc. + */ + public void cleanup() throws IOException {} + + /** * Continue wait and print status if job goes to ready or final state. * @param jobName * @throws IOException @@ -80,5 +85,6 @@ public abstract class JobMonitor { throw new IOException(e); } } +cleanup(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bee5bf86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobMonitor.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobMonitor.java
hadoop git commit: YARN-9001. [Submarine] Use AppAdminClient instead of ServiceClient to sumbit jobs. (Zac Zhou via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 9da6054ca -> fcbd205cc YARN-9001. [Submarine] Use AppAdminClient instead of ServiceClient to sumbit jobs. (Zac Zhou via wangda) Change-Id: Ic3d6c1e439df9cdf74448b345b925343224efe51 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcbd205c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcbd205c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcbd205c Branch: refs/heads/trunk Commit: fcbd205cc35e7411ac33860c78b9e1e70697bb4a Parents: 9da6054 Author: Wangda Tan Authored: Tue Nov 13 13:13:27 2018 -0800 Committer: Wangda Tan Committed: Tue Nov 13 13:13:27 2018 -0800 -- .../yarn/service/client/ServiceClient.java | 1 + .../submarine/runtimes/common/JobMonitor.java | 6 +++ .../yarnservice/YarnServiceJobMonitor.java | 27 + .../yarnservice/YarnServiceJobSubmitter.java| 42 ++-- .../runtimes/yarnservice/YarnServiceUtils.java | 15 +++ .../yarnservice/TestYarnServiceRunJobCli.java | 12 +++--- .../submarine/common/MockClientContext.java | 1 - 7 files changed, 80 insertions(+), 24 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcbd205c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 0bc5a2c..713d890 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -1552,6 +1552,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, LOG.info("Service {} does not have an application ID", serviceName); return appSpec; } +appSpec.setId(currentAppId.toString()); ApplicationReport appReport = yarnClient.getApplicationReport(currentAppId); appSpec.setState(convertState(appReport.getYarnApplicationState())); ApplicationTimeout lifetime = http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcbd205c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java index c81393b..35e21fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java @@ -48,6 +48,11 @@ public abstract class JobMonitor { throws IOException, YarnException; /** + * Cleanup AppAdminClient, etc. + */ + public void cleanup() throws IOException {} + + /** * Continue wait and print status if job goes to ready or final state. * @param jobName * @throws IOException @@ -80,5 +85,6 @@ public abstract class JobMonitor { throw new IOException(e); } } +cleanup(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcbd205c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobMonitor.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobMonitor.java index fab
hadoop git commit: MAPREDUCE-7158. Inefficient Flush Logic in JobHistory EventWriter. (Zichen Sun via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.1 c97f8b6d9 -> abd9d93a5 MAPREDUCE-7158. Inefficient Flush Logic in JobHistory EventWriter. (Zichen Sun via wangda) Change-Id: I99ace87980da03bb35a8012cea7218d602a8817a (cherry picked from commit 762a56cc64bc07d57f94e253920534b8e049f238) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/abd9d93a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/abd9d93a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/abd9d93a Branch: refs/heads/branch-3.1 Commit: abd9d93a5544a3172c9d9a48385c2ce7fc13fa34 Parents: c97f8b6 Author: Wangda Tan Authored: Tue Nov 13 11:25:41 2018 -0800 Committer: Wangda Tan Committed: Tue Nov 13 11:36:04 2018 -0800 -- .../java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java| 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/abd9d93a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java index 46e4f1a..b07c676 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java @@ -93,7 +93,6 @@ public class EventWriter { wrapper.setType(event.getEventType()); wrapper.setEvent(event.getDatum()); writer.write(wrapper, encoder); -encoder.flush(); if (this.jsonOutput) { out.writeBytes("\n"); } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: MAPREDUCE-7158. Inefficient Flush Logic in JobHistory EventWriter. (Zichen Sun via wangda)
MAPREDUCE-7158. Inefficient Flush Logic in JobHistory EventWriter. (Zichen Sun via wangda) Change-Id: I99ace87980da03bb35a8012cea7218d602a8817a (cherry picked from commit 762a56cc64bc07d57f94e253920534b8e049f238) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f214af74 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f214af74 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f214af74 Branch: refs/heads/branch-3.2 Commit: f214af74b29beb4aa76fde30d81cfb6e8c59f694 Parents: ea75304 Author: Wangda Tan Authored: Tue Nov 13 11:25:41 2018 -0800 Committer: Wangda Tan Committed: Tue Nov 13 11:35:13 2018 -0800 -- .../java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java| 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f214af74/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java index 46e4f1a..b07c676 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java @@ -93,7 +93,6 @@ public class EventWriter { wrapper.setType(event.getEventType()); wrapper.setEvent(event.getDatum()); writer.write(wrapper, encoder); -encoder.flush(); if (this.jsonOutput) { out.writeBytes("\n"); } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/2] hadoop git commit: YARN-8918. [Submarine] Correct method usage of str.subString in CliUtils. (Zhankun Tang via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.2 86deff0a0 -> f214af74b YARN-8918. [Submarine] Correct method usage of str.subString in CliUtils. (Zhankun Tang via wangda) Change-Id: Id1f11dbab3aa838dee3a0ec8b8fd5dc32f5dd946 (cherry picked from commit 076b795b2e82d7f89cc91e0a8513c7081ee8b930) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea753046 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea753046 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea753046 Branch: refs/heads/branch-3.2 Commit: ea753046b0f418dc850db618f1e2d8b4edeb3f81 Parents: 86deff0 Author: Wangda Tan Authored: Tue Nov 13 11:24:15 2018 -0800 Committer: Wangda Tan Committed: Tue Nov 13 11:35:06 2018 -0800 -- .../java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea753046/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java index 546c6eb..bfdfa9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java @@ -72,7 +72,7 @@ public class CliUtils { resourcesStr = resourcesStr.substring(1); } if (resourcesStr.endsWith("]")) { - resourcesStr = resourcesStr.substring(0, resourcesStr.length()); + resourcesStr = resourcesStr.substring(0, resourcesStr.length() - 1); } for (String resource : resourcesStr.trim().split(",")) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: MAPREDUCE-7158. Inefficient Flush Logic in JobHistory EventWriter. (Zichen Sun via wangda)
MAPREDUCE-7158. Inefficient Flush Logic in JobHistory EventWriter. (Zichen Sun via wangda) Change-Id: I99ace87980da03bb35a8012cea7218d602a8817a Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/762a56cc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/762a56cc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/762a56cc Branch: refs/heads/trunk Commit: 762a56cc64bc07d57f94e253920534b8e049f238 Parents: 076b795 Author: Wangda Tan Authored: Tue Nov 13 11:25:41 2018 -0800 Committer: Wangda Tan Committed: Tue Nov 13 11:25:41 2018 -0800 -- .../java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java| 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/762a56cc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java index 46e4f1a..b07c676 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java @@ -93,7 +93,6 @@ public class EventWriter { wrapper.setType(event.getEventType()); wrapper.setEvent(event.getDatum()); writer.write(wrapper, encoder); -encoder.flush(); if (this.jsonOutput) { out.writeBytes("\n"); } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/2] hadoop git commit: YARN-8918. [Submarine] Correct method usage of str.subString in CliUtils. (Zhankun Tang via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 671fd6524 -> 762a56cc6 YARN-8918. [Submarine] Correct method usage of str.subString in CliUtils. (Zhankun Tang via wangda) Change-Id: Id1f11dbab3aa838dee3a0ec8b8fd5dc32f5dd946 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/076b795b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/076b795b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/076b795b Branch: refs/heads/trunk Commit: 076b795b2e82d7f89cc91e0a8513c7081ee8b930 Parents: 671fd65 Author: Wangda Tan Authored: Tue Nov 13 11:24:15 2018 -0800 Committer: Wangda Tan Committed: Tue Nov 13 11:24:15 2018 -0800 -- .../java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/076b795b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java index 05e830f..f85c82a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java @@ -72,7 +72,7 @@ public class CliUtils { resourcesStr = resourcesStr.substring(1); } if (resourcesStr.endsWith("]")) { - resourcesStr = resourcesStr.substring(0, resourcesStr.length()); + resourcesStr = resourcesStr.substring(0, resourcesStr.length() - 1); } for (String resource : resourcesStr.trim().split(",")) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-8908. Fix errors in yarn-default.xml related to GPU/FPGA. (Zhankun Tang via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.1 bad5d81ac -> c11d2341a YARN-8908. Fix errors in yarn-default.xml related to GPU/FPGA. (Zhankun Tang via wangda) Change-Id: I46f511a9b51b65b069e15d5b827530110c41edab (cherry picked from commit 071b9e19aa2c17bb9fa046c6ddabd6b8f0cc6d14) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c11d2341 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c11d2341 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c11d2341 Branch: refs/heads/branch-3.1 Commit: c11d2341afe1da8a98c1d0f048147dc664fc8ab3 Parents: bad5d81 Author: Wangda Tan Authored: Fri Oct 19 09:44:00 2018 -0700 Committer: Wangda Tan Committed: Fri Oct 19 09:55:27 2018 -0700 -- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 5 +++-- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c11d2341/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 412fd02..fd68804 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3634,7 +3634,7 @@ Enable additional discovery/isolation of resources on the NodeManager, split by comma. By default, this is empty. - Acceptable values: { "yarn-io/gpu", "yarn-io/fpga"}. + Acceptable values: { "yarn.io/gpu", "yarn.io/fpga"}. yarn.nodemanager.resource-plugins @@ -3715,9 +3715,10 @@ At present, since we can only configure one major number in c-e.cfg, FPGA device is identified by their minor device number. A common approach to get minor device number of FPGA is using "aocl diagnose" and check uevent with device name. + A sample manual value for this is like "0,1" yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices -0,1 +auto http://git-wip-us.apache.org/repos/asf/hadoop/blob/c11d2341/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md index d98bb15..bc0589a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md @@ -53,7 +53,7 @@ Use following property to configure `DominantResourceCalculator` (In `capacity-s ``` yarn.nodemanager.resource-plugins -yarn-io/fpga +yarn.io/fpga ``` - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: YARN-8916. Define a constant docker string in ContainerRuntimeConstants.java for better maintainability. (Zhankun Tang via wangda)
YARN-8916. Define a constant docker string in ContainerRuntimeConstants.java for better maintainability. (Zhankun Tang via wangda) Change-Id: I1349e740037f81afdbe30edbe741f20e88fd0a90 (cherry picked from commit 5e02b4915b876eab943b30cf728cfc26ba832188) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ed9e185 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ed9e185 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ed9e185 Branch: refs/heads/branch-3.2 Commit: 9ed9e185d75ba303e04fd4a675c59dbc28670faf Parents: bd08554 Author: Wangda Tan Authored: Fri Oct 19 09:44:45 2018 -0700 Committer: Wangda Tan Committed: Fri Oct 19 09:50:18 2018 -0700 -- .../linux/runtime/DockerLinuxContainerRuntime.java | 3 ++- .../runtime/ContainerRuntimeConstants.java | 4 .../containermanager/container/TestContainer.java | 3 ++- .../linux/resources/gpu/TestGpuResourceHandler.java | 3 ++- .../runtime/TestDelegatingLinuxContainerRuntime.java| 5 +++-- .../linux/runtime/TestDockerContainerRuntime.java | 12 6 files changed, 21 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9e185/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index c86ed53..91a348d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -266,7 +266,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { if (type == null) { type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE); } -return type != null && type.equals("docker"); +return type != null && type.equals( +ContainerRuntimeConstants.CONTAINER_RUNTIME_DOCKER); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9e185/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java index 4473856..a8b295d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java @@ -30,4 +30,8 @@ public class ContainerRuntimeConstants { @Private public static final String ENV_CONTAINER_TYPE = "YARN_CONTAINER_RUNTIME_TYPE"; + + @Private + public static final String CONTAINER_RUNTIME_DOCKER = + "docker"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9e185/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/
[1/2] hadoop git commit: YARN-8908. Fix errors in yarn-default.xml related to GPU/FPGA. (Zhankun Tang via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.2 ef073547d -> 9ed9e185d YARN-8908. Fix errors in yarn-default.xml related to GPU/FPGA. (Zhankun Tang via wangda) Change-Id: I46f511a9b51b65b069e15d5b827530110c41edab (cherry picked from commit 071b9e19aa2c17bb9fa046c6ddabd6b8f0cc6d14) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd08554a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd08554a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd08554a Branch: refs/heads/branch-3.2 Commit: bd08554aa4220d0a5922cb08b4eaa246d09e7854 Parents: ef07354 Author: Wangda Tan Authored: Fri Oct 19 09:44:00 2018 -0700 Committer: Wangda Tan Committed: Fri Oct 19 09:50:12 2018 -0700 -- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 5 +++-- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd08554a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index e6f7b37..ba5a96e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3764,7 +3764,7 @@ Enable additional discovery/isolation of resources on the NodeManager, split by comma. By default, this is empty. - Acceptable values: { "yarn-io/gpu", "yarn-io/fpga"}. + Acceptable values: { "yarn.io/gpu", "yarn.io/fpga"}. yarn.nodemanager.resource-plugins @@ -3845,9 +3845,10 @@ At present, since we can only configure one major number in c-e.cfg, FPGA device is identified by their minor device number. A common approach to get minor device number of FPGA is using "aocl diagnose" and check uevent with device name. + A sample manual value for this is like "0,1" yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices -0,1 +auto http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd08554a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md index d98bb15..bc0589a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md @@ -53,7 +53,7 @@ Use following property to configure `DominantResourceCalculator` (In `capacity-s ``` yarn.nodemanager.resource-plugins -yarn-io/fpga +yarn.io/fpga ``` - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: YARN-8908. Fix errors in yarn-default.xml related to GPU/FPGA. (Zhankun Tang via wangda)
YARN-8908. Fix errors in yarn-default.xml related to GPU/FPGA. (Zhankun Tang via wangda) Change-Id: I46f511a9b51b65b069e15d5b827530110c41edab Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/071b9e19 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/071b9e19 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/071b9e19 Branch: refs/heads/trunk Commit: 071b9e19aa2c17bb9fa046c6ddabd6b8f0cc6d14 Parents: cd05791 Author: Wangda Tan Authored: Fri Oct 19 09:44:00 2018 -0700 Committer: Wangda Tan Committed: Fri Oct 19 09:49:26 2018 -0700 -- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 5 +++-- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/071b9e19/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 2ea6bfc..1360e73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3787,7 +3787,7 @@ Enable additional discovery/isolation of resources on the NodeManager, split by comma. By default, this is empty. - Acceptable values: { "yarn-io/gpu", "yarn-io/fpga"}. + Acceptable values: { "yarn.io/gpu", "yarn.io/fpga"}. yarn.nodemanager.resource-plugins @@ -3868,9 +3868,10 @@ At present, since we can only configure one major number in c-e.cfg, FPGA device is identified by their minor device number. A common approach to get minor device number of FPGA is using "aocl diagnose" and check uevent with device name. + A sample manual value for this is like "0,1" yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices -0,1 +auto http://git-wip-us.apache.org/repos/asf/hadoop/blob/071b9e19/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md index d98bb15..bc0589a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md @@ -53,7 +53,7 @@ Use following property to configure `DominantResourceCalculator` (In `capacity-s ``` yarn.nodemanager.resource-plugins -yarn-io/fpga +yarn.io/fpga ``` - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/2] hadoop git commit: YARN-8916. Define a constant docker string in ContainerRuntimeConstants.java for better maintainability. (Zhankun Tang via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk cd057918d -> 5e02b4915 YARN-8916. Define a constant docker string in ContainerRuntimeConstants.java for better maintainability. (Zhankun Tang via wangda) Change-Id: I1349e740037f81afdbe30edbe741f20e88fd0a90 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e02b491 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e02b491 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e02b491 Branch: refs/heads/trunk Commit: 5e02b4915b876eab943b30cf728cfc26ba832188 Parents: 071b9e1 Author: Wangda Tan Authored: Fri Oct 19 09:44:45 2018 -0700 Committer: Wangda Tan Committed: Fri Oct 19 09:49:26 2018 -0700 -- .../linux/runtime/DockerLinuxContainerRuntime.java | 3 ++- .../runtime/ContainerRuntimeConstants.java | 4 .../containermanager/container/TestContainer.java | 3 ++- .../linux/resources/gpu/TestGpuResourceHandler.java | 3 ++- .../runtime/TestDelegatingLinuxContainerRuntime.java| 5 +++-- .../linux/runtime/TestDockerContainerRuntime.java | 12 6 files changed, 21 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e02b491/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 21660c1..436c0ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -266,7 +266,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { if (type == null) { type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE); } -return type != null && type.equals("docker"); +return type != null && type.equals( +ContainerRuntimeConstants.CONTAINER_RUNTIME_DOCKER); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e02b491/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java index 4473856..a8b295d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java @@ -30,4 +30,8 @@ public class ContainerRuntimeConstants { @Private public static final String ENV_CONTAINER_TYPE = "YARN_CONTAINER_RUNTIME_TYPE"; + + @Private + public static final String CONTAINER_RUNTIME_DOCKER = + "docker"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e02b491/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodema
[2/2] hadoop git commit: YARN-6098. Add documentation for Delete Queue. (Suma Shivaprasad via wangda)
YARN-6098. Add documentation for Delete Queue. (Suma Shivaprasad via wangda) Change-Id: I23bf7ca0acbe1168a7f805ceccdd62fe41bfeb35 (cherry picked from commit bfb88b10f46a265aa38ab3e1d87b6a0a99d94be8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d381f322 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d381f322 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d381f322 Branch: refs/heads/branch-3.1 Commit: d381f32219dbe27b629ba3bea3424f9648287c97 Parents: 46baafe Author: Wangda Tan Authored: Thu Oct 18 10:55:39 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 12:29:28 2018 -0700 -- .../src/site/markdown/CapacityScheduler.md| 14 +- 1 file changed, 13 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d381f322/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 9857010..75d7954 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -54,7 +54,7 @@ The `CapacityScheduler` supports the following features: * **Operability** -* Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime. +* Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime unless the queue is STOPPED and nhas no pending/running apps. * Drain applications - Administrators can *stop* queues at runtime to ensure that while existing applications run to completion, no new applications can be submitted. If a queue is in `STOPPED` state, new applications cannot be submitted to *itself* or *any of its child queues*. Existing applications continue to completion, thus the queue can be *drained* gracefully. Administrators can also *start* the stopped queues. @@ -430,6 +430,18 @@ Changing queue/scheduler properties and adding/removing queues can be done in tw $ vi $HADOOP_CONF_DIR/capacity-scheduler.xml $ $HADOOP_YARN_HOME/bin/yarn rmadmin -refreshQueues + Deleting queue via file + + Step 1: Stop the queue + + Before deleting a leaf queue, the leaf queue should not have any running/pending apps and has to BE STOPPED by changing `yarn.scheduler.capacity..state`. See the + [Queue Administration & Permissions](CapacityScheduler.html#Queue Properties) section. + Before deleting a parent queue, all its child queues should not have any running/pending apps and have to BE STOPPED. The parent queue also needs to be STOPPED + + Step 2: Delete the queue + + Remove the queue configurations from the file and run refresh as described above + ### Changing queue configuration via API Editing by API uses a backing store for the scheduler configuration. To enable this, the following parameters can be configured in yarn-site.xml. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/2] hadoop git commit: YARN-8896. Limit the maximum number of container assignments per heartbeat. (Zhankun Tang via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.1 3cb52958b -> d381f3221 YARN-8896. Limit the maximum number of container assignments per heartbeat. (Zhankun Tang via wangda) Change-Id: I6e72f8362bd7f5c2a844cb9e3c4732492314e9f1 (cherry picked from commit 780be14f07df2a3ed6273b96ae857c278fd72718) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46baafed Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46baafed Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46baafed Branch: refs/heads/branch-3.1 Commit: 46baafedf13446f5c1b973d667f437bf7158d3c2 Parents: 3cb5295 Author: Wangda Tan Authored: Thu Oct 18 10:58:21 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 12:29:19 2018 -0700 -- .../scheduler/capacity/CapacitySchedulerConfiguration.java | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/46baafed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 5c9904b..547d131 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -333,8 +333,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public static final String MAX_ASSIGN_PER_HEARTBEAT = PREFIX + "per-node-heartbeat.maximum-container-assignments"; + /** + * Avoid potential risk that greedy assign multiple may involve + * */ @Private - public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1; + public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = 100; /** Configuring absolute min/max resources in a queue. **/ @Private - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[3/3] hadoop git commit: YARN-8896. Limit the maximum number of container assignments per heartbeat. (Zhankun Tang via wangda)
YARN-8896. Limit the maximum number of container assignments per heartbeat. (Zhankun Tang via wangda) Change-Id: I6e72f8362bd7f5c2a844cb9e3c4732492314e9f1 (cherry picked from commit 780be14f07df2a3ed6273b96ae857c278fd72718) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7252f8e1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7252f8e1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7252f8e1 Branch: refs/heads/branch-3.2 Commit: 7252f8e117bacdd3b746021d4b5d6b9c944813bd Parents: 5f7ed04 Author: Wangda Tan Authored: Thu Oct 18 10:58:21 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 12:12:19 2018 -0700 -- .../scheduler/capacity/CapacitySchedulerConfiguration.java | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7252f8e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index b937ae7..08380f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -336,8 +336,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public static final String MAX_ASSIGN_PER_HEARTBEAT = PREFIX + "per-node-heartbeat.maximum-container-assignments"; + /** + * Avoid potential risk that greedy assign multiple may involve + * */ @Private - public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1; + public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = 100; /** Configuring absolute min/max resources in a queue. **/ @Private - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/3] hadoop git commit: YARN-8456. Fix a configuration handling bug when user leave FPGA discover executable path configuration default but set OpenCL SDK path environment variable. (Zhankun Tang via
YARN-8456. Fix a configuration handling bug when user leave FPGA discover executable path configuration default but set OpenCL SDK path environment variable. (Zhankun Tang via wangda) Change-Id: Iff150ea98ba0c60d448474fd940eb121afce6965 (cherry picked from commit a457a8951a1b35f06811c40443ca44bb9c698c30) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f7ed043 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f7ed043 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f7ed043 Branch: refs/heads/branch-3.2 Commit: 5f7ed043c8e8f9d964a719d076a9d3d207d4cea7 Parents: a7785d2 Author: Wangda Tan Authored: Thu Oct 18 10:57:11 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 12:12:14 2018 -0700 -- .../fpga/IntelFpgaOpenclPlugin.java | 4 +- .../resourceplugin/fpga/TestFpgaDiscoverer.java | 64 ++-- 2 files changed, 61 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f7ed043/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java index 2d6cf6f..f8fb6d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java @@ -124,7 +124,9 @@ public class IntelFpgaOpenclPlugin implements AbstractFpgaVendorPlugin { } else { binaryPath = new File(pluginDefaultPreferredPath + "/bin", pluginDefaultBinaryName); if (binaryPath.exists()) { - pathToExecutable = pluginDefaultPreferredPath; + pathToExecutable = binaryPath.getAbsolutePath(); + LOG.info("Succeed in finding FPGA discoverer executable: " + + pathToExecutable); } else { pathToExecutable = pluginDefaultBinaryName; LOG.warn("Failed to find FPGA discoverer executable in " + http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f7ed043/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java index 87fb4e9..d5bcdb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java @@ -32,9 +32,8 @@ import org.junit.Test; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.lang.reflect.Field; +import java.util.*; import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyString; @@ -60,8 +59,41 @@ public class TestFpgaDiscoverer { f.mkdirs(); } + // A dirty hack to modify the env of the current JVM itself - Dirty, but + // should be okay for testing. + @SuppressWarnings({ "rawtypes", "unchecked" }) + private static void setNewEnvironmentHack(Map newenv) + throws Exception { +try { + Class cl = Class.forName("java.lang.ProcessEnvironment"); + Field field = cl.getDeclaredField("theEnvironment"
[1/3] hadoop git commit: YARN-6098. Add documentation for Delete Queue. (Suma Shivaprasad via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.2 4681e17a3 -> 7252f8e11 YARN-6098. Add documentation for Delete Queue. (Suma Shivaprasad via wangda) Change-Id: I23bf7ca0acbe1168a7f805ceccdd62fe41bfeb35 (cherry picked from commit bfb88b10f46a265aa38ab3e1d87b6a0a99d94be8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a7785d27 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a7785d27 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a7785d27 Branch: refs/heads/branch-3.2 Commit: a7785d2764767ffed7acb0cef7d2894c9a43618a Parents: 4681e17 Author: Wangda Tan Authored: Thu Oct 18 10:55:39 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 12:12:09 2018 -0700 -- .../src/site/markdown/CapacityScheduler.md| 14 +- 1 file changed, 13 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7785d27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 5ac1d0a..c2d4ce4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -54,7 +54,7 @@ The `CapacityScheduler` supports the following features: * **Operability** -* Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime. +* Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime unless the queue is STOPPED and nhas no pending/running apps. * Drain applications - Administrators can *stop* queues at runtime to ensure that while existing applications run to completion, no new applications can be submitted. If a queue is in `STOPPED` state, new applications cannot be submitted to *itself* or *any of its child queues*. Existing applications continue to completion, thus the queue can be *drained* gracefully. Administrators can also *start* the stopped queues. @@ -441,6 +441,18 @@ Changing queue/scheduler properties and adding/removing queues can be done in tw $ vi $HADOOP_CONF_DIR/capacity-scheduler.xml $ $HADOOP_YARN_HOME/bin/yarn rmadmin -refreshQueues + Deleting queue via file + + Step 1: Stop the queue + + Before deleting a leaf queue, the leaf queue should not have any running/pending apps and has to BE STOPPED by changing `yarn.scheduler.capacity..state`. See the + [Queue Administration & Permissions](CapacityScheduler.html#Queue Properties) section. + Before deleting a parent queue, all its child queues should not have any running/pending apps and have to BE STOPPED. The parent queue also needs to be STOPPED + + Step 2: Delete the queue + + Remove the queue configurations from the file and run refresh as described above + ### Changing queue configuration via API Editing by API uses a backing store for the scheduler configuration. To enable this, the following parameters can be configured in yarn-site.xml. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[3/3] hadoop git commit: YARN-8896. Limit the maximum number of container assignments per heartbeat. (Zhankun Tang via wangda)
YARN-8896. Limit the maximum number of container assignments per heartbeat. (Zhankun Tang via wangda) Change-Id: I6e72f8362bd7f5c2a844cb9e3c4732492314e9f1 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/780be14f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/780be14f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/780be14f Branch: refs/heads/trunk Commit: 780be14f07df2a3ed6273b96ae857c278fd72718 Parents: a457a89 Author: Wangda Tan Authored: Thu Oct 18 10:58:21 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 10:58:21 2018 -0700 -- .../scheduler/capacity/CapacitySchedulerConfiguration.java | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/780be14f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index b937ae7..08380f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -336,8 +336,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public static final String MAX_ASSIGN_PER_HEARTBEAT = PREFIX + "per-node-heartbeat.maximum-container-assignments"; + /** + * Avoid potential risk that greedy assign multiple may involve + * */ @Private - public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1; + public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = 100; /** Configuring absolute min/max resources in a queue. **/ @Private - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/3] hadoop git commit: YARN-8456. Fix a configuration handling bug when user leave FPGA discover executable path configuration default but set OpenCL SDK path environment variable. (Zhankun Tang via
YARN-8456. Fix a configuration handling bug when user leave FPGA discover executable path configuration default but set OpenCL SDK path environment variable. (Zhankun Tang via wangda) Change-Id: Iff150ea98ba0c60d448474fd940eb121afce6965 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a457a895 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a457a895 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a457a895 Branch: refs/heads/trunk Commit: a457a8951a1b35f06811c40443ca44bb9c698c30 Parents: bfb88b1 Author: Wangda Tan Authored: Thu Oct 18 10:57:11 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 10:57:11 2018 -0700 -- .../fpga/IntelFpgaOpenclPlugin.java | 4 +- .../resourceplugin/fpga/TestFpgaDiscoverer.java | 64 ++-- 2 files changed, 61 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a457a895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java index 2d6cf6f..f8fb6d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java @@ -124,7 +124,9 @@ public class IntelFpgaOpenclPlugin implements AbstractFpgaVendorPlugin { } else { binaryPath = new File(pluginDefaultPreferredPath + "/bin", pluginDefaultBinaryName); if (binaryPath.exists()) { - pathToExecutable = pluginDefaultPreferredPath; + pathToExecutable = binaryPath.getAbsolutePath(); + LOG.info("Succeed in finding FPGA discoverer executable: " + + pathToExecutable); } else { pathToExecutable = pluginDefaultBinaryName; LOG.warn("Failed to find FPGA discoverer executable in " + http://git-wip-us.apache.org/repos/asf/hadoop/blob/a457a895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java index 87fb4e9..d5bcdb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java @@ -32,9 +32,8 @@ import org.junit.Test; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.lang.reflect.Field; +import java.util.*; import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyString; @@ -60,8 +59,41 @@ public class TestFpgaDiscoverer { f.mkdirs(); } + // A dirty hack to modify the env of the current JVM itself - Dirty, but + // should be okay for testing. + @SuppressWarnings({ "rawtypes", "unchecked" }) + private static void setNewEnvironmentHack(Map newenv) + throws Exception { +try { + Class cl = Class.forName("java.lang.ProcessEnvironment"); + Field field = cl.getDeclaredField("theEnvironment"); + field.setAccessible(true); + Map env = (Map) field.get(null
[1/3] hadoop git commit: YARN-6098. Add documentation for Delete Queue. (Suma Shivaprasad via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 0e56c883c -> 780be14f0 YARN-6098. Add documentation for Delete Queue. (Suma Shivaprasad via wangda) Change-Id: I23bf7ca0acbe1168a7f805ceccdd62fe41bfeb35 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bfb88b10 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bfb88b10 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bfb88b10 Branch: refs/heads/trunk Commit: bfb88b10f46a265aa38ab3e1d87b6a0a99d94be8 Parents: 0e56c88 Author: Wangda Tan Authored: Thu Oct 18 10:55:39 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 10:55:39 2018 -0700 -- .../src/site/markdown/CapacityScheduler.md| 14 +- 1 file changed, 13 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb88b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 5ac1d0a..c2d4ce4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -54,7 +54,7 @@ The `CapacityScheduler` supports the following features: * **Operability** -* Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime. +* Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime unless the queue is STOPPED and nhas no pending/running apps. * Drain applications - Administrators can *stop* queues at runtime to ensure that while existing applications run to completion, no new applications can be submitted. If a queue is in `STOPPED` state, new applications cannot be submitted to *itself* or *any of its child queues*. Existing applications continue to completion, thus the queue can be *drained* gracefully. Administrators can also *start* the stopped queues. @@ -441,6 +441,18 @@ Changing queue/scheduler properties and adding/removing queues can be done in tw $ vi $HADOOP_CONF_DIR/capacity-scheduler.xml $ $HADOOP_YARN_HOME/bin/yarn rmadmin -refreshQueues + Deleting queue via file + + Step 1: Stop the queue + + Before deleting a leaf queue, the leaf queue should not have any running/pending apps and has to BE STOPPED by changing `yarn.scheduler.capacity..state`. See the + [Queue Administration & Permissions](CapacityScheduler.html#Queue Properties) section. + Before deleting a parent queue, all its child queues should not have any running/pending apps and have to BE STOPPED. The parent queue also needs to be STOPPED + + Step 2: Delete the queue + + Remove the queue configurations from the file and run refresh as described above + ### Changing queue configuration via API Editing by API uses a backing store for the scheduler configuration. To enable this, the following parameters can be configured in yarn-site.xml. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.1 30526f2c5 -> 3e1894c37 YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda) Change-Id: I8bc8622936861b8d6de3e42a0b75af86ad8a3961 (cherry picked from commit 538250db26ce0b261bb74053348cddfc2d65cf52) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e1894c3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e1894c3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e1894c3 Branch: refs/heads/branch-3.1 Commit: 3e1894c3766e42d6756758d0d90d97eefc2ef331 Parents: 30526f2 Author: Wangda Tan Authored: Tue Oct 16 13:41:17 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 14:36:36 2018 -0700 -- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e1894c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md index 609ebe1..4c9daed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md @@ -52,4 +52,4 @@ Open your browser, go to `rm-address:8088/ui2` and try it! Notes - -- This UI framework is not verified under security environment, please use with caution under security environment. +This UI framework is verified under security environment as well. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[4/5] hadoop git commit: YARN-8798. [Submarine] Job should not be submitted if --input_path option is missing. (Zhankun Tang via wangda)
YARN-8798. [Submarine] Job should not be submitted if --input_path option is missing. (Zhankun Tang via wangda) Change-Id: I7ae0e44eb5179b04a6ac861ec1c65f3b18c38f0f (cherry picked from commit 143d74775b2b62884090fdd88874134b9eab2888) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa23c8d7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa23c8d7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa23c8d7 Branch: refs/heads/branch-3.2 Commit: aa23c8d7396430f3863d56e2ea0d94ac3fdb34a0 Parents: 920401b Author: Wangda Tan Authored: Tue Oct 16 13:39:34 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:53:03 2018 -0700 -- .../client/cli/param/RunJobParameters.java | 6 +++ .../client/cli/TestRunJobCliParsing.java| 39 2 files changed, 45 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa23c8d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java index 92a1883..d923e0f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java @@ -62,6 +62,12 @@ public class RunJobParameters extends RunParameters { if (parsedCommandLine.getOptionValue(CliConstants.N_WORKERS) != null) { nWorkers = Integer.parseInt( parsedCommandLine.getOptionValue(CliConstants.N_WORKERS)); + // Only check null value. + // Training job shouldn't ignore INPUT_PATH option + // But if nWorkers is 0, INPUT_PATH can be ignored because user can only run Tensorboard + if (null == input && 0 != nWorkers) { +throw new ParseException("\"--" + CliConstants.INPUT_PATH + "\" is absent"); + } } int nPS = 0; http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa23c8d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java index 295d6a8..240de06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.submarine.client.cli; +import org.apache.commons.cli.ParseException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; @@ -136,6 +137,44 @@ public class TestRunJobCliParsing { } @Test + public void testNoInputPathOptionSpecified() throws Exception { +RunJobCli runJobCli = new RunJobCli(getMockClientContext()); +String expectedErrorMessage = "\"--" + CliConstants.INPUT_PATH + "\" is absent"; +String actualMessage = ""; +try { + runJobCli.run( + new String[]{"--name", "my-job", "--docker_image", "tf-docker:1.1.0", + "--checkpoint_path", "hdfs://output", + "--num_workers", "1", "--worker_launch_cmd", "python run-job.py", + "--worker_resources", "memory=4g,vcores=2", "--tensorboard", + "true", "--verbose", "--wait_job_finish"}); +} catch (ParseException e)
[2/5] hadoop git commit: YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/920401b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh new file mode 100644 index 000..7b3c2a9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description check install user +## @audience public +## @stabilitystable +function check_install_user() +{ + if [[ $(id -u) -ne 0 ]];then +echo "This script must be run with a ROOT user!" +exit # don't call exit_install() + fi +} + +## @description exit install +## @audience public +## @stabilitystable +function exit_install() +{ + echo "Exit the installation!" | tee -a $LOG + exit $1 +} + +## @description Check if the IP address format is correct +## @audience public +## @stabilitystable +function valid_ip() +{ + local ip=$1 + local stat=1 + + if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then +OIFS=$IFS +IFS='.' +ip=($ip) +IFS=$OIFS + +if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]; then + stat=$? +fi + fi + + return $stat +} + +## @description Check if the configuration file configuration is correct +## @audience public +## @stabilitystable +function check_install_conf() +{ + echo "Check if the configuration file configuration is correct ..." | tee -a $LOG + + # check etcd conf + hostCount=${#ETCD_HOSTS[@]} + if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2 +echo "Number of nodes = [$hostCount], must be configured to be greater than or equal to 3 servers! " | tee -a $LOG +exit_install + fi + for ip in ${ETCD_HOSTS[@]} + do +if ! valid_ip $ip; then + echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG + exit_install +fi + done + echo "Check if the configuration file configuration is correct [ Done ]" | tee -a $LOG +} + +## @description index by EtcdHosts list +## @audience public +## @stabilitystable +function indexByEtcdHosts() { + index=0 + while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do +if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then + echo $index + return +fi +let "index++" + done + echo "" +} + +## @description get local IP +## @audience public +## @stabilitystable +function getLocalIP() +{ + local _ip _myip _line _nl=$'\n' + while IFS=$': \t' read -a _line ;do + [ -z "${_line%inet}" ] && + _ip=${_line[${#_line[1]}>4?1:2]} && + [ "${_ip#127.0.0.1}" ] && _myip=$_ip +done< <(LANG=C /sbin/ifconfig) + printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip +} + +## @description get ip list +## @audience public +## @stabilitystable +function get_ip_list() +{ + array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[ \t]*//g' | cut -d ' ' -f2) + + for ip in ${array[@]} + do +LOCAL_HOST_IP_LIST+=(${ip}) + done +} - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/5] hadoop git commit: YARN-8875. [Submarine] Add documentation for submarine installation script details. (Xun Liu via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.2 c19864d8c -> 80ee60c88 YARN-8875. [Submarine] Add documentation for submarine installation script details. (Xun Liu via wangda) Change-Id: I1c8d39c394e5a30f967ea514919835b951f2c124 (cherry picked from commit ed08dd3b0c9cec20373e8ca4e34d6526bd759943) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8610c207 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8610c207 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8610c207 Branch: refs/heads/branch-3.2 Commit: 8610c20724141563f34025ba8d67c8d2ed2c54e5 Parents: c19864d Author: Wangda Tan Authored: Tue Oct 16 13:36:09 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:52:53 2018 -0700 -- .../src/site/markdown/HowToInstall.md | 36 +++ .../src/site/markdown/Index.md | 4 +- .../src/site/markdown/InstallationGuide.md | 205 +++ .../src/site/markdown/InstallationScriptCN.md | 242 ++ .../src/site/markdown/InstallationScriptEN.md | 250 +++ .../src/site/markdown/TestAndTroubleshooting.md | 165 .../resources/images/submarine-installer.gif| Bin 0 -> 546547 bytes 7 files changed, 724 insertions(+), 178 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8610c207/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md new file mode 100644 index 000..05d87c1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md @@ -0,0 +1,36 @@ + + +# How to Install Dependencies + +Submarine project uses YARN Service, Docker container, and GPU (when GPU hardware available and properly configured). + +That means as an admin, you have to properly setup YARN Service related dependencies, including: +- YARN Registry DNS + +Docker related dependencies, including: +- Docker binary with expected versions. +- Docker network which allows Docker container can talk to each other across different nodes. + +And when GPU wanna to be used: +- GPU Driver. +- Nvidia-docker. + +For your convenience, we provided installation documents to help you to setup your environment. You can always choose to have them installed in your own way. + +Use Submarine installer to install dependencies: [EN](InstallationScriptEN.html) [CN](InstallationScriptCN.html) + +Alternatively, you can follow manual install dependencies: [EN](InstallationGuide.html) [CN](InstallationGuideChineseVersion.html) + +Once you have installed dependencies, please follow following guide to [TestAndTroubleshooting](TestAndTroubleshooting.html). \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/8610c207/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md index 0006f6c..baeaa15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md @@ -41,6 +41,4 @@ Click below contents if you want to understand more. - [Developer guide](DeveloperGuide.html) -- [Installation guide](InstallationGuide.html) - -- [Installation guide Chinese version](InstallationGuideChineseVersion.html) +- [Installation guides](HowToInstall.html) http://git-wip-us.apache.org/repos/asf/hadoop/blob/8610c207/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md index d4f4269..4ef2bda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applic
[5/5] hadoop git commit: YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda)
YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda) Change-Id: I8bc8622936861b8d6de3e42a0b75af86ad8a3961 (cherry picked from commit 538250db26ce0b261bb74053348cddfc2d65cf52) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80ee60c8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80ee60c8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80ee60c8 Branch: refs/heads/branch-3.2 Commit: 80ee60c88ca91381fecc6c2aaaf0804a39f38a8d Parents: aa23c8d Author: Wangda Tan Authored: Tue Oct 16 13:41:17 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:53:08 2018 -0700 -- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/80ee60c8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md index 609ebe1..4c9daed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md @@ -52,4 +52,4 @@ Open your browser, go to `rm-address:8088/ui2` and try it! Notes - -- This UI framework is not verified under security environment, please use with caution under security environment. +This UI framework is verified under security environment as well. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[3/5] hadoop git commit: YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)
YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda) Change-Id: I46e8d9fd32c7745c313030da62da41486a77b3ea (cherry picked from commit 46d6e0016610ced51a76189daeb3ad0e3dbbf94c) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/920401b2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/920401b2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/920401b2 Branch: refs/heads/branch-3.2 Commit: 920401b2966c2dcd7cd1b3dfd7a764f08412323f Parents: 8610c20 Author: Wangda Tan Authored: Tue Oct 16 13:36:59 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:52:58 2018 -0700 -- .../resources/assemblies/hadoop-yarn-dist.xml | 8 + .../installation/install.conf | 74 .../installation/install.sh | 116 + .../package/calico/calico-node.service | 50 +++ .../installation/package/calico/calicoctl.cfg | 22 + .../installation/package/docker/daemon.json | 23 + .../installation/package/docker/docker.service | 35 ++ .../installation/package/etcd/etcd.service | 40 ++ .../package/hadoop/container-executor.cfg | 41 ++ .../installation/package/submarine/submarine.sh | 25 ++ .../installation/scripts/calico.sh | 224 ++ .../installation/scripts/docker.sh | 166 +++ .../installation/scripts/download-server.sh | 42 ++ .../installation/scripts/environment.sh | 213 + .../installation/scripts/etcd.sh| 152 +++ .../installation/scripts/hadoop.sh | 117 + .../installation/scripts/menu.sh| 444 +++ .../installation/scripts/nvidia-docker.sh | 99 + .../installation/scripts/nvidia.sh | 120 + .../installation/scripts/submarine.sh | 38 ++ .../installation/scripts/utils.sh | 123 + 21 files changed, 2172 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/920401b2/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index a2ea08c..9c401e8 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -66,6 +66,14 @@ 0755 + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation + /share/hadoop/yarn/submarine-installer + +**/* + + 0755 + + hadoop-yarn/conf etc/hadoop http://git-wip-us.apache.org/repos/asf/hadoop/blob/920401b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf new file mode 100644 index 000..82dcf61 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# install config # + +# DNS +LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf +YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address + +# etcd hosts list +ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) + +# docker registry ip:port +DOCKER_REGISTRY="10.120.196.232:5000" + +# Start the http download service on the specified server, +# Will download all the dependencies in the http server, +# Run the install script on other servers. +# Automatically download dependencies from http, +# Solve
[2/5] hadoop git commit: YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh new file mode 100644 index 000..7b3c2a9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description check install user +## @audience public +## @stabilitystable +function check_install_user() +{ + if [[ $(id -u) -ne 0 ]];then +echo "This script must be run with a ROOT user!" +exit # don't call exit_install() + fi +} + +## @description exit install +## @audience public +## @stabilitystable +function exit_install() +{ + echo "Exit the installation!" | tee -a $LOG + exit $1 +} + +## @description Check if the IP address format is correct +## @audience public +## @stabilitystable +function valid_ip() +{ + local ip=$1 + local stat=1 + + if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then +OIFS=$IFS +IFS='.' +ip=($ip) +IFS=$OIFS + +if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]; then + stat=$? +fi + fi + + return $stat +} + +## @description Check if the configuration file configuration is correct +## @audience public +## @stabilitystable +function check_install_conf() +{ + echo "Check if the configuration file configuration is correct ..." | tee -a $LOG + + # check etcd conf + hostCount=${#ETCD_HOSTS[@]} + if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2 +echo "Number of nodes = [$hostCount], must be configured to be greater than or equal to 3 servers! " | tee -a $LOG +exit_install + fi + for ip in ${ETCD_HOSTS[@]} + do +if ! valid_ip $ip; then + echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG + exit_install +fi + done + echo "Check if the configuration file configuration is correct [ Done ]" | tee -a $LOG +} + +## @description index by EtcdHosts list +## @audience public +## @stabilitystable +function indexByEtcdHosts() { + index=0 + while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do +if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then + echo $index + return +fi +let "index++" + done + echo "" +} + +## @description get local IP +## @audience public +## @stabilitystable +function getLocalIP() +{ + local _ip _myip _line _nl=$'\n' + while IFS=$': \t' read -a _line ;do + [ -z "${_line%inet}" ] && + _ip=${_line[${#_line[1]}>4?1:2]} && + [ "${_ip#127.0.0.1}" ] && _myip=$_ip +done< <(LANG=C /sbin/ifconfig) + printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip +} + +## @description get ip list +## @audience public +## @stabilitystable +function get_ip_list() +{ + array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[ \t]*//g' | cut -d ' ' -f2) + + for ip in ${array[@]} + do +LOCAL_HOST_IP_LIST+=(${ip}) + done +} - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[5/5] hadoop git commit: YARN-8798. [Submarine] Job should not be submitted if --input_path option is missing. (Zhankun Tang via wangda)
YARN-8798. [Submarine] Job should not be submitted if --input_path option is missing. (Zhankun Tang via wangda) Change-Id: I7ae0e44eb5179b04a6ac861ec1c65f3b18c38f0f Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/143d7477 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/143d7477 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/143d7477 Branch: refs/heads/trunk Commit: 143d74775b2b62884090fdd88874134b9eab2888 Parents: 46d6e00 Author: Wangda Tan Authored: Tue Oct 16 13:39:34 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:51:02 2018 -0700 -- .../client/cli/param/RunJobParameters.java | 6 +++ .../client/cli/TestRunJobCliParsing.java| 39 2 files changed, 45 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/143d7477/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java index 92a1883..d923e0f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java @@ -62,6 +62,12 @@ public class RunJobParameters extends RunParameters { if (parsedCommandLine.getOptionValue(CliConstants.N_WORKERS) != null) { nWorkers = Integer.parseInt( parsedCommandLine.getOptionValue(CliConstants.N_WORKERS)); + // Only check null value. + // Training job shouldn't ignore INPUT_PATH option + // But if nWorkers is 0, INPUT_PATH can be ignored because user can only run Tensorboard + if (null == input && 0 != nWorkers) { +throw new ParseException("\"--" + CliConstants.INPUT_PATH + "\" is absent"); + } } int nPS = 0; http://git-wip-us.apache.org/repos/asf/hadoop/blob/143d7477/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java index 295d6a8..240de06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.submarine.client.cli; +import org.apache.commons.cli.ParseException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; @@ -136,6 +137,44 @@ public class TestRunJobCliParsing { } @Test + public void testNoInputPathOptionSpecified() throws Exception { +RunJobCli runJobCli = new RunJobCli(getMockClientContext()); +String expectedErrorMessage = "\"--" + CliConstants.INPUT_PATH + "\" is absent"; +String actualMessage = ""; +try { + runJobCli.run( + new String[]{"--name", "my-job", "--docker_image", "tf-docker:1.1.0", + "--checkpoint_path", "hdfs://output", + "--num_workers", "1", "--worker_launch_cmd", "python run-job.py", + "--worker_resources", "memory=4g,vcores=2", "--tensorboard", + "true", "--verbose", "--wait_job_finish"}); +} catch (ParseException e) { + actualMessage = e.getMessage(); + e.printStackTrace(); +} +Assert.assertEquals(expe
[3/5] hadoop git commit: YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)
YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda) Change-Id: I46e8d9fd32c7745c313030da62da41486a77b3ea Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46d6e001 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46d6e001 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46d6e001 Branch: refs/heads/trunk Commit: 46d6e0016610ced51a76189daeb3ad0e3dbbf94c Parents: ed08dd3 Author: Wangda Tan Authored: Tue Oct 16 13:36:59 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:51:02 2018 -0700 -- .../resources/assemblies/hadoop-yarn-dist.xml | 8 + .../installation/install.conf | 74 .../installation/install.sh | 116 + .../package/calico/calico-node.service | 50 +++ .../installation/package/calico/calicoctl.cfg | 22 + .../installation/package/docker/daemon.json | 23 + .../installation/package/docker/docker.service | 35 ++ .../installation/package/etcd/etcd.service | 40 ++ .../package/hadoop/container-executor.cfg | 41 ++ .../installation/package/submarine/submarine.sh | 25 ++ .../installation/scripts/calico.sh | 224 ++ .../installation/scripts/docker.sh | 166 +++ .../installation/scripts/download-server.sh | 42 ++ .../installation/scripts/environment.sh | 213 + .../installation/scripts/etcd.sh| 152 +++ .../installation/scripts/hadoop.sh | 117 + .../installation/scripts/menu.sh| 444 +++ .../installation/scripts/nvidia-docker.sh | 99 + .../installation/scripts/nvidia.sh | 120 + .../installation/scripts/submarine.sh | 38 ++ .../installation/scripts/utils.sh | 123 + 21 files changed, 2172 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index a2ea08c..9c401e8 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -66,6 +66,14 @@ 0755 + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation + /share/hadoop/yarn/submarine-installer + +**/* + + 0755 + + hadoop-yarn/conf etc/hadoop http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf new file mode 100644 index 000..82dcf61 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# install config # + +# DNS +LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf +YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address + +# etcd hosts list +ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) + +# docker registry ip:port +DOCKER_REGISTRY="10.120.196.232:5000" + +# Start the http download service on the specified server, +# Will download all the dependencies in the http server, +# Run the install script on other servers. +# Automatically download dependencies from http, +# Solve the problem that all servers are slow to download online. +# At the same tim
[4/5] hadoop git commit: YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda)
YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda) Change-Id: I8bc8622936861b8d6de3e42a0b75af86ad8a3961 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/538250db Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/538250db Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/538250db Branch: refs/heads/trunk Commit: 538250db26ce0b261bb74053348cddfc2d65cf52 Parents: 143d747 Author: Wangda Tan Authored: Tue Oct 16 13:41:17 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:51:02 2018 -0700 -- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/538250db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md index 609ebe1..4c9daed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md @@ -52,4 +52,4 @@ Open your browser, go to `rm-address:8088/ui2` and try it! Notes - -- This UI framework is not verified under security environment, please use with caution under security environment. +This UI framework is verified under security environment as well. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/5] hadoop git commit: YARN-8875. [Submarine] Add documentation for submarine installation script details. (Xun Liu via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk babd1449b -> 538250db2 YARN-8875. [Submarine] Add documentation for submarine installation script details. (Xun Liu via wangda) Change-Id: I1c8d39c394e5a30f967ea514919835b951f2c124 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed08dd3b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed08dd3b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed08dd3b Branch: refs/heads/trunk Commit: ed08dd3b0c9cec20373e8ca4e34d6526bd759943 Parents: babd144 Author: Wangda Tan Authored: Tue Oct 16 13:36:09 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:51:01 2018 -0700 -- .../src/site/markdown/HowToInstall.md | 36 +++ .../src/site/markdown/Index.md | 4 +- .../src/site/markdown/InstallationGuide.md | 205 +++ .../src/site/markdown/InstallationScriptCN.md | 242 ++ .../src/site/markdown/InstallationScriptEN.md | 250 +++ .../src/site/markdown/TestAndTroubleshooting.md | 165 .../resources/images/submarine-installer.gif| Bin 0 -> 546547 bytes 7 files changed, 724 insertions(+), 178 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed08dd3b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md new file mode 100644 index 000..05d87c1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md @@ -0,0 +1,36 @@ + + +# How to Install Dependencies + +Submarine project uses YARN Service, Docker container, and GPU (when GPU hardware available and properly configured). + +That means as an admin, you have to properly setup YARN Service related dependencies, including: +- YARN Registry DNS + +Docker related dependencies, including: +- Docker binary with expected versions. +- Docker network which allows Docker container can talk to each other across different nodes. + +And when GPU wanna to be used: +- GPU Driver. +- Nvidia-docker. + +For your convenience, we provided installation documents to help you to setup your environment. You can always choose to have them installed in your own way. + +Use Submarine installer to install dependencies: [EN](InstallationScriptEN.html) [CN](InstallationScriptCN.html) + +Alternatively, you can follow manual install dependencies: [EN](InstallationGuide.html) [CN](InstallationGuideChineseVersion.html) + +Once you have installed dependencies, please follow following guide to [TestAndTroubleshooting](TestAndTroubleshooting.html). \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed08dd3b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md index 0006f6c..baeaa15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md @@ -41,6 +41,4 @@ Click below contents if you want to understand more. - [Developer guide](DeveloperGuide.html) -- [Installation guide](InstallationGuide.html) - -- [Installation guide Chinese version](InstallationGuideChineseVersion.html) +- [Installation guides](HowToInstall.html) http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed08dd3b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md index d4f4269..4ef2bda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md @@ -16,9
hadoop git commit: YARN-8852. Add documentation for submarine installation details. (Zac Zhou via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.2 f6a73d181 -> 8e8b74872 YARN-8852. Add documentation for submarine installation details. (Zac Zhou via wangda) Change-Id: If5681d1ef37ff5dc916735eeef15a6120173d653 (cherry picked from commit a23ea68b9747eae9b176f908bb04b76d30fe3795) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e8b7487 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e8b7487 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e8b7487 Branch: refs/heads/branch-3.2 Commit: 8e8b74872498077a1c3568b5f55b4e215669bad4 Parents: f6a73d1 Author: Wangda Tan Authored: Tue Oct 9 10:18:00 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 9 10:19:16 2018 -0700 -- .../src/site/markdown/Index.md | 4 + .../src/site/markdown/InstallationGuide.md | 760 +++ .../markdown/InstallationGuideChineseVersion.md | 757 ++ 3 files changed, 1521 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e8b7487/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md index 0b78a87..0006f6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md @@ -40,3 +40,7 @@ Click below contents if you want to understand more. - [How to write Dockerfile for Submarine jobs](WriteDockerfile.html) - [Developer guide](DeveloperGuide.html) + +- [Installation guide](InstallationGuide.html) + +- [Installation guide Chinese version](InstallationGuideChineseVersion.html) http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e8b7487/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md new file mode 100644 index 000..d4f4269 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md @@ -0,0 +1,760 @@ + + +# Submarine Installation Guide + +## Prerequisites + +### Operating System + +The operating system and kernel versions we used are as shown in the following table, which should be minimum required versions: + +| Enviroment | Verion | +| -- | -- | +| Operating System | centos-release-7-3.1611.el7.centos.x86_64 | +| Kernal | 3.10.0-514.el7.x86_64 | + +### User & Group + +As there are some specific users and groups need to be created to install hadoop/docker. Please create them if they are missing. + +``` +adduser hdfs +adduser mapred +adduser yarn +addgroup hadoop +usermod -aG hdfs,hadoop hdfs +usermod -aG mapred,hadoop mapred +usermod -aG yarn,hadoop yarn +usermod -aG hdfs,hadoop hadoop +groupadd docker +usermod -aG docker yarn +usermod -aG docker hadoop +``` + +### GCC Version + +Check the version of GCC tool + +```bash +gcc --version +gcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-11) +# install if needed +yum install gcc make g++ +``` + +### Kernel header & Kernel devel + +```bash +# Approach 1ï¼ +yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r) +# Approach 2ï¼ +wget http://vault.centos.org/7.3.1611/os/x86_64/Packages/kernel-headers-3.10.0-514.el7.x86_64.rpm +rpm -ivh kernel-headers-3.10.0-514.el7.x86_64.rpm +``` + +### GPU Servers + +``` +lspci | grep -i nvidia + +# If the server has gpus, you can get info like thisï¼ +04:00.0 3D controller: NVIDIA Corporation Device 1b38 (rev a1) +82:00.0 3D controller: NVIDIA Corporation Device 1b38 (rev a1) +``` + + + +### Nvidia Driver Installation + +If nvidia driver/cuda has been installed before, They should be uninstalled firstly. + +``` +# uninstall cudaï¼ +sudo /usr/local/cuda-10.0/bin/uninstall_cuda_10.0.pl + +# uninstall nvidia-driverï¼ +sudo /usr/bin/nvidia-uninstall +``` + +To check GPU version, install nvidia-detect + +``` +yum install nvidia-detect +# run 'nvidia-detect -v' to get reqired nvidia driver versionï¼ +nvidia-detect -v +Probing for supported NVIDIA devices... +[10de:13bb] NVIDIA Corporation GM107GL [Quadro K620] +This device requir
hadoop git commit: YARN-8852. Add documentation for submarine installation details. (Zac Zhou via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 600438bcd -> a23ea68b9 YARN-8852. Add documentation for submarine installation details. (Zac Zhou via wangda) Change-Id: If5681d1ef37ff5dc916735eeef15a6120173d653 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a23ea68b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a23ea68b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a23ea68b Branch: refs/heads/trunk Commit: a23ea68b9747eae9b176f908bb04b76d30fe3795 Parents: 600438b Author: Wangda Tan Authored: Tue Oct 9 10:18:00 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 9 10:18:27 2018 -0700 -- .../src/site/markdown/Index.md | 4 + .../src/site/markdown/InstallationGuide.md | 760 +++ .../markdown/InstallationGuideChineseVersion.md | 757 ++ 3 files changed, 1521 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ea68b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md index 0b78a87..0006f6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md @@ -40,3 +40,7 @@ Click below contents if you want to understand more. - [How to write Dockerfile for Submarine jobs](WriteDockerfile.html) - [Developer guide](DeveloperGuide.html) + +- [Installation guide](InstallationGuide.html) + +- [Installation guide Chinese version](InstallationGuideChineseVersion.html) http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ea68b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md new file mode 100644 index 000..d4f4269 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md @@ -0,0 +1,760 @@ + + +# Submarine Installation Guide + +## Prerequisites + +### Operating System + +The operating system and kernel versions we used are as shown in the following table, which should be minimum required versions: + +| Enviroment | Verion | +| -- | -- | +| Operating System | centos-release-7-3.1611.el7.centos.x86_64 | +| Kernal | 3.10.0-514.el7.x86_64 | + +### User & Group + +As there are some specific users and groups need to be created to install hadoop/docker. Please create them if they are missing. + +``` +adduser hdfs +adduser mapred +adduser yarn +addgroup hadoop +usermod -aG hdfs,hadoop hdfs +usermod -aG mapred,hadoop mapred +usermod -aG yarn,hadoop yarn +usermod -aG hdfs,hadoop hadoop +groupadd docker +usermod -aG docker yarn +usermod -aG docker hadoop +``` + +### GCC Version + +Check the version of GCC tool + +```bash +gcc --version +gcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-11) +# install if needed +yum install gcc make g++ +``` + +### Kernel header & Kernel devel + +```bash +# Approach 1ï¼ +yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r) +# Approach 2ï¼ +wget http://vault.centos.org/7.3.1611/os/x86_64/Packages/kernel-headers-3.10.0-514.el7.x86_64.rpm +rpm -ivh kernel-headers-3.10.0-514.el7.x86_64.rpm +``` + +### GPU Servers + +``` +lspci | grep -i nvidia + +# If the server has gpus, you can get info like thisï¼ +04:00.0 3D controller: NVIDIA Corporation Device 1b38 (rev a1) +82:00.0 3D controller: NVIDIA Corporation Device 1b38 (rev a1) +``` + + + +### Nvidia Driver Installation + +If nvidia driver/cuda has been installed before, They should be uninstalled firstly. + +``` +# uninstall cudaï¼ +sudo /usr/local/cuda-10.0/bin/uninstall_cuda_10.0.pl + +# uninstall nvidia-driverï¼ +sudo /usr/bin/nvidia-uninstall +``` + +To check GPU version, install nvidia-detect + +``` +yum install nvidia-detect +# run 'nvidia-detect -v' to get reqired nvidia driver versionï¼ +nvidia-detect -v +Probing for supported NVIDIA devices... +[10de:13bb] NVIDIA Corporation GM107GL [Quadro K620] +This device requires the current 390.87 NVIDIA driver kmod-nvidia +[8086:1912] Intel Corporation HD G
hadoop git commit: YARN-8844. TestNMProxy unit test is failing. (Eric Yang via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 6926fd0ec -> 2e9913caf YARN-8844. TestNMProxy unit test is failing. (Eric Yang via wangda) Change-Id: I241fa8701b6f1dbcad87fd2e9a429e32e7aa40f5 (cherry picked from commit b3ac8869338faeabedbc27e95e3166c830a9a761) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e9913ca Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e9913ca Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e9913ca Branch: refs/heads/trunk Commit: 2e9913caf2a0ba24c323c26dd978492bd398f2e5 Parents: 6926fd0 Author: Wangda Tan Authored: Thu Oct 4 10:48:47 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 4 10:53:39 2018 -0700 -- .../yarn/server/nodemanager/containermanager/TestNMProxy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e9913ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java index e6a7a02..5f023f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java @@ -161,7 +161,7 @@ public class TestNMProxy extends BaseContainerManagerTest { IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 100); // connect to some dummy address so that it can trigger // connection failure and RPC level retires. -newConf.set(YarnConfiguration.NM_ADDRESS, "1234"); +newConf.set(YarnConfiguration.NM_ADDRESS, "0.0.0.0:1234"); ContainerManagementProtocol proxy = getNMProxy(newConf); try { proxy.startContainers(allRequests); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: YARN-8844. TestNMProxy unit test is failing. (Eric Yang via wangda)
YARN-8844. TestNMProxy unit test is failing. (Eric Yang via wangda) Change-Id: I241fa8701b6f1dbcad87fd2e9a429e32e7aa40f5 (cherry picked from commit b3ac8869338faeabedbc27e95e3166c830a9a761) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86a1ad44 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86a1ad44 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86a1ad44 Branch: refs/heads/branch-3.1 Commit: 86a1ad442855cfa350c8bf791e85b09ddbecd0bd Parents: 30f50a0 Author: Wangda Tan Authored: Thu Oct 4 10:48:47 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 4 10:49:29 2018 -0700 -- .../yarn/server/nodemanager/containermanager/TestNMProxy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a1ad44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java index e6a7a02..5f023f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java @@ -161,7 +161,7 @@ public class TestNMProxy extends BaseContainerManagerTest { IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 100); // connect to some dummy address so that it can trigger // connection failure and RPC level retires. -newConf.set(YarnConfiguration.NM_ADDRESS, "1234"); +newConf.set(YarnConfiguration.NM_ADDRESS, "0.0.0.0:1234"); ContainerManagementProtocol proxy = getNMProxy(newConf); try { proxy.startContainers(allRequests); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/2] hadoop git commit: YARN-8758. Support getting PreemptionMessage when using AMRMClientAsyn. (Zian Chen via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.1 1a890b17b -> 86a1ad442 YARN-8758. Support getting PreemptionMessage when using AMRMClientAsyn. (Zian Chen via wangda) Change-Id: Ibf5d165f49957b582eeadeb41dc285c84d2f05e7 (cherry picked from commit 6926fd0ec634df2576bbc9f45e9636b99260db72) (cherry picked from commit abe4a8e5d82e70ce991e2830f20eba9f25a2491a) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30f50a06 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30f50a06 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30f50a06 Branch: refs/heads/branch-3.1 Commit: 30f50a0682d3ddb682a15ccf20f7fb797c17d18a Parents: 1a890b1 Author: Wangda Tan Authored: Thu Oct 4 10:31:33 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 4 10:49:23 2018 -0700 -- .../hadoop/yarn/client/api/async/AMRMClientAsync.java| 11 +++ .../yarn/client/api/async/impl/AMRMClientAsyncImpl.java | 9 + 2 files changed, 20 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/30f50a06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java index 3dd2f71..c9f4e5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.ContainerUpdateType; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.PreemptionMessage; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest; import org.apache.hadoop.yarn.api.records.Resource; @@ -553,6 +554,16 @@ extends AbstractService { public void onRequestsRejected( List rejectedSchedulingRequests) { } + +/** + * Called when the RM responds to a heartbeat with preemption message + * @param preemptionMessage + */ +@Public +@Unstable +public void onPreemptionMessageReceived( +PreemptionMessage preemptionMessage) { +} } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/30f50a06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java index 3cf2c34..922b185 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.PreemptionMessage; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest; import org.apache.hadoop.yarn.api.records.Resource; @@ -401,6 +402,14 @@ extends AMRMClientAsync { handler.onContainersAllocated(allocated); } + PreemptionMessage preemptionMessage = response.getPreemptionMessage(); + if (preemptionMessage != null) { +if (handler instanceof AMRMClientAsync.AbstractCallbackHandler) { + ((AMRMClientAsync.AbstractCallbackHandler) handler) + .onPreemptionMessageReceived(preemptionMessage); +} + } + if (!response.getContainersFromPreviousAttempts().isEmpty()) { if (handler instanceof AMRMClientAsync.AbstractCallbackHandler) { ((AMRMClientAsync.AbstractCallbackHandler) hand
[2/2] hadoop git commit: YARN-8844. TestNMProxy unit test is failing. (Eric Yang via wangda)
YARN-8844. TestNMProxy unit test is failing. (Eric Yang via wangda) Change-Id: I241fa8701b6f1dbcad87fd2e9a429e32e7aa40f5 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3ac8869 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3ac8869 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3ac8869 Branch: refs/heads/branch-3.2 Commit: b3ac8869338faeabedbc27e95e3166c830a9a761 Parents: abe4a8e Author: Wangda Tan Authored: Thu Oct 4 10:48:47 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 4 10:48:47 2018 -0700 -- .../yarn/server/nodemanager/containermanager/TestNMProxy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ac8869/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java index e6a7a02..5f023f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java @@ -161,7 +161,7 @@ public class TestNMProxy extends BaseContainerManagerTest { IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 100); // connect to some dummy address so that it can trigger // connection failure and RPC level retires. -newConf.set(YarnConfiguration.NM_ADDRESS, "1234"); +newConf.set(YarnConfiguration.NM_ADDRESS, "0.0.0.0:1234"); ContainerManagementProtocol proxy = getNMProxy(newConf); try { proxy.startContainers(allRequests); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/2] hadoop git commit: YARN-8758. Support getting PreemptionMessage when using AMRMClientAsyn. (Zian Chen via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.2 e185ae2d1 -> b3ac88693 YARN-8758. Support getting PreemptionMessage when using AMRMClientAsyn. (Zian Chen via wangda) Change-Id: Ibf5d165f49957b582eeadeb41dc285c84d2f05e7 (cherry picked from commit 6926fd0ec634df2576bbc9f45e9636b99260db72) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/abe4a8e5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/abe4a8e5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/abe4a8e5 Branch: refs/heads/branch-3.2 Commit: abe4a8e5d82e70ce991e2830f20eba9f25a2491a Parents: e185ae2 Author: Wangda Tan Authored: Thu Oct 4 10:31:33 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 4 10:43:16 2018 -0700 -- .../hadoop/yarn/client/api/async/AMRMClientAsync.java| 11 +++ .../yarn/client/api/async/impl/AMRMClientAsyncImpl.java | 9 + 2 files changed, 20 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/abe4a8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java index 3dd2f71..c9f4e5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.ContainerUpdateType; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.PreemptionMessage; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest; import org.apache.hadoop.yarn.api.records.Resource; @@ -553,6 +554,16 @@ extends AbstractService { public void onRequestsRejected( List rejectedSchedulingRequests) { } + +/** + * Called when the RM responds to a heartbeat with preemption message + * @param preemptionMessage + */ +@Public +@Unstable +public void onPreemptionMessageReceived( +PreemptionMessage preemptionMessage) { +} } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/abe4a8e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java index 3cf2c34..922b185 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.PreemptionMessage; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest; import org.apache.hadoop.yarn.api.records.Resource; @@ -401,6 +402,14 @@ extends AMRMClientAsync { handler.onContainersAllocated(allocated); } + PreemptionMessage preemptionMessage = response.getPreemptionMessage(); + if (preemptionMessage != null) { +if (handler instanceof AMRMClientAsync.AbstractCallbackHandler) { + ((AMRMClientAsync.AbstractCallbackHandler) handler) + .onPreemptionMessageReceived(preemptionMessage); +} + } + if (!response.getContainersFromPreviousAttempts().isEmpty()) { if (handler instanceof AMRMClientAsync.AbstractCallbackHandler) { ((AMRMClientAsync.AbstractCallbackHandler) handler) -
hadoop git commit: YARN-8758. Support getting PreemptionMessage when using AMRMClientAsyn. (Zian Chen via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 81f635f47 -> 6926fd0ec YARN-8758. Support getting PreemptionMessage when using AMRMClientAsyn. (Zian Chen via wangda) Change-Id: Ibf5d165f49957b582eeadeb41dc285c84d2f05e7 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6926fd0e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6926fd0e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6926fd0e Branch: refs/heads/trunk Commit: 6926fd0ec634df2576bbc9f45e9636b99260db72 Parents: 81f635f Author: Wangda Tan Authored: Thu Oct 4 10:31:33 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 4 10:31:33 2018 -0700 -- .../hadoop/yarn/client/api/async/AMRMClientAsync.java| 11 +++ .../yarn/client/api/async/impl/AMRMClientAsyncImpl.java | 9 + 2 files changed, 20 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6926fd0e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java index 3dd2f71..c9f4e5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.ContainerUpdateType; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.PreemptionMessage; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest; import org.apache.hadoop.yarn.api.records.Resource; @@ -553,6 +554,16 @@ extends AbstractService { public void onRequestsRejected( List rejectedSchedulingRequests) { } + +/** + * Called when the RM responds to a heartbeat with preemption message + * @param preemptionMessage + */ +@Public +@Unstable +public void onPreemptionMessageReceived( +PreemptionMessage preemptionMessage) { +} } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/6926fd0e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java index 3cf2c34..922b185 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.PreemptionMessage; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest; import org.apache.hadoop.yarn.api.records.Resource; @@ -401,6 +402,14 @@ extends AMRMClientAsync { handler.onContainersAllocated(allocated); } + PreemptionMessage preemptionMessage = response.getPreemptionMessage(); + if (preemptionMessage != null) { +if (handler instanceof AMRMClientAsync.AbstractCallbackHandler) { + ((AMRMClientAsync.AbstractCallbackHandler) handler) + .onPreemptionMessageReceived(preemptionMessage); +} + } + if (!response.getContainersFromPreviousAttempts().isEmpty()) { if (handler instanceof AMRMClientAsync.AbstractCallbackHandler) { ((AMRMClientAsync.AbstractCallbackHandler) handler) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additio
hadoop git commit: MAPREDUCE-7125. JobResourceUploader creates LocalFileSystem when it's not necessary. (Peter Cseh via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.1 0196ffaa2 -> daa14bf06 MAPREDUCE-7125. JobResourceUploader creates LocalFileSystem when it's not necessary. (Peter Cseh via wangda) Change-Id: I1aa720ed03739f6f4abeec46f6068e2ab332987a (cherry picked from commit 230d90c5b9cc0d6e74bf16b1cf6685a163a24d48) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/daa14bf0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/daa14bf0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/daa14bf0 Branch: refs/heads/branch-3.1 Commit: daa14bf06a56cc03513c4039c6da902af4d8b9d7 Parents: 0196ffa Author: Wangda Tan Authored: Mon Sep 24 13:30:02 2018 -0700 Committer: Wangda Tan Committed: Mon Sep 24 13:53:51 2018 -0700 -- .../main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa14bf0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java index 1dbecb6..e106a54 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java @@ -855,8 +855,8 @@ class JobResourceUploader { throw new IllegalArgumentException(e); } Path path = new Path(pathURI); -FileSystem localFs = FileSystem.getLocal(conf); if (pathURI.getScheme() == null) { + FileSystem localFs = FileSystem.getLocal(conf); // default to the local file system // check if the file exists or not first localFs.getFileStatus(path); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org