[hadoop] 02/02: YARN-10587. Fix AutoCreateLeafQueueCreation cap related caculation when in absolute mode. (Qi Zhu via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 4f008153ef5fca9e1f71ebc7069c502e803ab1e8
Author: Wangda Tan
AuthorDate: Thu Jan 21 09:31:14 2021 -0800
YARN-10587. Fix AutoCreateLeafQueueCreation cap related caculation when in
absolute mode. (Qi Zhu via wangda)
Change-Id: I8889c393a9f08b633c72a2e873c6f45ca12cc82b
---
.../GuaranteedOrZeroCapacityOverTimePolicy.java | 9 ++---
.../capacity/TestAbsoluteResourceConfiguration.java | 12 +++-
.../capacity/TestAbsoluteResourceWithAutoQueue.java | 12 ++--
3 files changed, 19 insertions(+), 14 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
index ab99317..a53fe4a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
@@ -19,6 +19,7 @@ package
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
.queuemanagement;
import
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.api.records.Resource;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
.QueueManagementDynamicEditPolicy;
@@ -624,7 +625,7 @@ public class GuaranteedOrZeroCapacityOverTimePolicy
QueueCapacities capacities = leafQueueEntitlements.get(
leafQueue.getQueuePath());
- updateToZeroCapacity(capacities, nodeLabel);
+ updateToZeroCapacity(capacities, nodeLabel, (LeafQueue)childQueue);
deactivatedQueues.put(leafQueue.getQueuePath(),
leafQueueTemplateCapacities);
}
@@ -822,7 +823,7 @@ public class GuaranteedOrZeroCapacityOverTimePolicy
updateCapacityFromTemplate(capacities, nodeLabel);
activate(leafQueue, nodeLabel);
} else{
- updateToZeroCapacity(capacities, nodeLabel);
+ updateToZeroCapacity(capacities, nodeLabel, leafQueue);
}
}
@@ -834,10 +835,12 @@ public class GuaranteedOrZeroCapacityOverTimePolicy
}
private void updateToZeroCapacity(QueueCapacities capacities,
- String nodeLabel) {
+ String nodeLabel, LeafQueue leafQueue) {
capacities.setCapacity(nodeLabel, 0.0f);
capacities.setMaximumCapacity(nodeLabel,
leafQueueTemplateCapacities.getMaximumCapacity(nodeLabel));
+leafQueue.getQueueResourceQuotas().
+setConfiguredMinResource(nodeLabel, Resource.newInstance(0, 0));
}
private void updateCapacityFromTemplate(QueueCapacities capacities,
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
index da13e18..97f9a65 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -231,8 +231,10 @@ public class TestAbsoluteResourceConfiguration {
d1.queueResourceQuotas.getEffectiveMaxResource());
/**
- * After adding child queue d2, adjustment happens and both d1 and d2
shares
- * resource of Managed Parent Queue
+ * After adding child queue d2, d1 + d2 > resource
+ * of Managed Parent queue, d2 will change to 0.
+ * d1 will occupy all entire resource
+ * of Managed Parent queue.
*/
AutoCreatedLeafQueue d2 = new AutoCreatedLeafQueue(cs, "d2", parentQueue);
cs.addQueue(d2);
@@ -240,9 +242,9 @@
[hadoop] branch trunk updated (f1766e5 -> 4f00815)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from f1766e5 YARN-10596. Allow static definition of childless ParentQueues with auto-queue-creation-v2 enabled. Contributed by Andras Gyori new b7384a8 YARN-10531. Be able to disable user limit factor for CapacityScheduler Leaf Queue. (Qi Zhu via wangda) new 4f00815 YARN-10587. Fix AutoCreateLeafQueueCreation cap related caculation when in absolute mode. (Qi Zhu via wangda) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../scheduler/capacity/AbstractCSQueue.java| 9 +- .../scheduler/capacity/LeafQueue.java | 26 - .../scheduler/capacity/ParentQueue.java| 4 +- .../scheduler/capacity/PlanQueue.java | 3 + .../scheduler/capacity/UsersManager.java | 12 ++- .../GuaranteedOrZeroCapacityOverTimePolicy.java| 9 +- .../TestAbsoluteResourceConfiguration.java | 12 ++- .../TestAbsoluteResourceWithAutoQueue.java | 12 +-- .../TestCapacitySchedulerNewQueueAutoCreation.java | 35 +++ .../scheduler/capacity/TestLeafQueue.java | 110 - 10 files changed, 209 insertions(+), 23 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 01/02: YARN-10531. Be able to disable user limit factor for CapacityScheduler Leaf Queue. (Qi Zhu via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit b7384a8d02ae3cace8c7b5eae84f8f3916a0b177
Author: Wangda Tan
AuthorDate: Thu Jan 21 09:27:37 2021 -0800
YARN-10531. Be able to disable user limit factor for CapacityScheduler Leaf
Queue. (Qi Zhu via wangda)
Change-Id: I670e5525619b320745254609c48e7e1afb084835
---
.../scheduler/capacity/AbstractCSQueue.java| 9 +-
.../scheduler/capacity/LeafQueue.java | 26 -
.../scheduler/capacity/ParentQueue.java| 4 +-
.../scheduler/capacity/PlanQueue.java | 3 +
.../scheduler/capacity/UsersManager.java | 12 ++-
.../TestCapacitySchedulerNewQueueAutoCreation.java | 35 +++
.../scheduler/capacity/TestLeafQueue.java | 110 -
7 files changed, 190 insertions(+), 9 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index bc3ff22..12ce05f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -1535,8 +1535,13 @@ public abstract class AbstractCSQueue implements CSQueue
{
leafQueue.setMaxApplications(maxApplications);
int maxApplicationsPerUser = Math.min(maxApplications,
- (int) (maxApplications * (leafQueue.getUsersManager().getUserLimit()
- / 100.0f) * leafQueue.getUsersManager().getUserLimitFactor()));
+ (int) (maxApplications
+ * (leafQueue.getUsersManager().getUserLimit() / 100.0f)
+ * leafQueue.getUsersManager().getUserLimitFactor()));
+ if (leafQueue.getUsersManager().getUserLimitFactor() == -1) {
+maxApplicationsPerUser = maxApplications;
+ }
+
leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser);
LOG.info("LeafQueue:" + leafQueue.getQueuePath() + ", maxApplications="
+ maxApplications + ", maxApplicationsPerUser="
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 15c321f..6bf8d0a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -708,16 +708,33 @@ public class LeafQueue extends AbstractCSQueue {
queueCapacities.getMaxAMResourcePercentage(nodePartition)
* effectiveUserLimit * usersManager.getUserLimitFactor(),
minimumAllocation);
+
+ if (getUserLimitFactor() == -1) {
+userAMLimit = Resources.multiplyAndNormalizeUp(
+resourceCalculator, queuePartitionResource,
+queueCapacities.getMaxAMResourcePercentage(nodePartition),
+minimumAllocation);
+ }
+
userAMLimit =
Resources.min(resourceCalculator, lastClusterResource,
userAMLimit,
Resources.clone(getAMResourceLimitPerPartition(nodePartition)));
- Resource preWeighteduserAMLimit = Resources.multiplyAndNormalizeUp(
+ Resource preWeighteduserAMLimit =
+ Resources.multiplyAndNormalizeUp(
resourceCalculator, queuePartitionResource,
queueCapacities.getMaxAMResourcePercentage(nodePartition)
* preWeightedUserLimit * usersManager.getUserLimitFactor(),
minimumAllocation);
+
+ if (getUserLimitFactor() == -1) {
+preWeighteduserAMLimit = Resources.multiplyAndNormalizeUp(
+resourceCalculator, queuePartitionResource,
+queueCapacities.getMaxAMResourcePercentage(nodePartition),
+minimumAllocation);
+ }
+
preWeighteduserAMLimit =
Resources.min(resourceCalculator, lastClusterResource,
preWeighteduserAMLimit,
@@ -1896,9 +191
[hadoop] branch trunk updated: YARN-10506. Update queue creation logic to use weight mode and allow the flexible static/dynamic creation. (Contributed by Qi Zhu, Andras Gyori)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3d46141 YARN-10506. Update queue creation logic to use weight mode
and allow the flexible static/dynamic creation. (Contributed by Qi Zhu, Andras
Gyori)
3d46141 is described below
commit 3d46141583e357ee12181fc9084ef6234bedf3f3
Author: Wangda Tan
AuthorDate: Fri Jan 15 14:20:08 2021 -0800
YARN-10506. Update queue creation logic to use weight mode and allow the
flexible static/dynamic creation. (Contributed by Qi Zhu, Andras Gyori)
Change-Id: I118862fd5e11ee6888275e2bcf667fedfa56c5d7
---
.../placement/ApplicationPlacementContext.java | 13 +
.../scheduler/capacity/AbstractCSQueue.java| 54 ++-
.../capacity/AbstractManagedParentQueue.java | 2 +-
.../scheduler/capacity/CSQueueUtils.java | 18 +-
.../scheduler/capacity/CapacityScheduler.java | 84 ++--
.../CapacitySchedulerAutoQueueHandler.java | 127 ++
.../capacity/CapacitySchedulerConfigValidator.java | 14 +-
.../capacity/CapacitySchedulerConfiguration.java | 27 +-
.../capacity/CapacitySchedulerQueueManager.java| 35 +-
.../scheduler/capacity/LeafQueue.java | 17 +-
.../scheduler/capacity/ParentQueue.java| 165 +++-
.../scheduler/capacity/PlanQueue.java | 3 +-
.../scheduler/capacity/QueueCapacities.java| 2 +-
.../TestCapacitySchedulerAutoQueueCreation.java| 25 +-
.../TestCapacitySchedulerNewQueueAutoCreation.java | 436 +
.../scheduler/capacity/TestLeafQueue.java | 4 +
16 files changed, 927 insertions(+), 99 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java
index f2f92b8..3ae9ac4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java
@@ -42,6 +42,10 @@ public class ApplicationPlacementContext {
return queue;
}
+ public void setQueue(String q) {
+queue = q;
+ }
+
public String getParentQueue() {
return parentQueue;
}
@@ -49,4 +53,13 @@ public class ApplicationPlacementContext {
public boolean hasParentQueue() {
return parentQueue != null;
}
+
+ public String getFullQueuePath() {
+if (parentQueue != null) {
+ return parentQueue + "." + queue;
+} else {
+ return queue;
+}
+ }
+
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 9e7b0d8..fd144f2 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -151,6 +151,14 @@ public abstract class AbstractCSQueue implements CSQueue {
private Map userWeights = new HashMap();
private int maxParallelApps;
+ // is it a dynamic queue?
+ private boolean dynamicQueue = false;
+
+ // When this queue has application submit to?
+ // This property only applies to dynamic queue,
+ // and will be used to check when the queue need to be removed.
+ private long lastSubmittedTimestamp;
+
public AbstractCSQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
this(cs, cs.getConfiguration(), queueName, parent, old);
@@ -172,7 +180,7 @@ public abstract class AbstractCSQueue implements CSQueue {
this.metrics = old != null ?
(CSQueueMetrics) old.getMetrics() :
CSQueueMetrics.forQueue(getQueuePath(), parent,
-cs.getConfiguration().getEnableUserMetrics(), cs.getConf());
+cs.getConfiguration().getEnableUserMetrics(), configuration);
this.csCo
[hadoop] branch trunk updated: YARN-10504. Implement weight mode in Capacity Scheduler. (Contributed by Wangda Tan, Benjamin Teke, zhuqi, Andras Gyori)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b0eec09 YARN-10504. Implement weight mode in Capacity Scheduler.
(Contributed by Wangda Tan, Benjamin Teke, zhuqi, Andras Gyori)
b0eec09 is described below
commit b0eec0909772cf92427957670da5630b1dd11da0
Author: Wangda Tan
AuthorDate: Mon Jan 11 17:46:09 2021 -0800
YARN-10504. Implement weight mode in Capacity Scheduler. (Contributed by
Wangda Tan, Benjamin Teke, zhuqi, Andras Gyori)
Change-Id: Ic49c730b0ab502ba86527fb662d25c4c8b1c2588
---
.../capacity/AbstractAutoCreatedLeafQueue.java | 5 -
.../scheduler/capacity/AbstractCSQueue.java| 242 +++--
.../scheduler/capacity/AutoCreatedLeafQueue.java | 16 -
.../scheduler/capacity/CSQueueUtils.java | 124 ++---
.../capacity/CapacitySchedulerConfiguration.java | 88 +++-
.../capacity/CapacitySchedulerQueueManager.java| 4 +
.../scheduler/capacity/LeafQueue.java | 50 +-
.../scheduler/capacity/ManagedParentQueue.java | 20 +-
.../scheduler/capacity/ParentQueue.java| 541 ++---
.../scheduler/capacity/QueueCapacities.java| 54 +-
.../scheduler/capacity/ReservationQueue.java | 4 +-
.../GuaranteedOrZeroCapacityOverTimePolicy.java| 29 ++
.../TestAbsoluteResourceConfiguration.java | 7 +-
.../TestAbsoluteResourceWithAutoQueue.java | 10 +-
.../capacity/TestCSMaxRunningAppsEnforcer.java | 12 +-
.../TestCapacitySchedulerAutoCreatedQueueBase.java | 12 +-
.../capacity/TestCapacitySchedulerWeightMode.java | 452 +
.../scheduler/capacity/TestLeafQueue.java | 11 +-
.../scheduler/capacity/TestParentQueue.java| 9 +-
.../scheduler/capacity/TestQueueCapacities.java| 21 +-
.../scheduler/capacity/TestQueueParsing.java | 55 ++-
.../scheduler/capacity/TestReservationQueue.java | 18 +-
.../resourcemanager/webapp/TestRMWebServices.java | 2 +-
.../TestRMWebServicesForCSWithPartitions.java | 16 +-
24 files changed, 1301 insertions(+), 501 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
index 2b22241..8d77334 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
@@ -128,9 +128,4 @@ public class AbstractAutoCreatedLeafQueue extends LeafQueue
{
writeLock.unlock();
}
}
-
- protected void setupConfigurableCapacities(QueueCapacities queueCapacities) {
-CSQueueUtils.updateAndCheckCapacitiesByLabel(getQueuePath(),
-queueCapacities, parent == null ? null : parent.getQueueCapacities());
- }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 8d22a36..9e7b0d8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -100,7 +100,7 @@ public abstract class AbstractCSQueue implements CSQueue {
String defaultLabelExpression;
private String multiNodeSortingPolicyName = null;
- Map acls =
+ Map acls =
new HashMap();
volatile boolean reservationsContinueLooking;
private volatile boolean preemptionDisabled;
@@ -112,7 +112,7 @@ public abstract class AbstractCSQueue implements CSQueue {
volatile ResourceUsage queueUsage;
private final boolean fullPathQueueNamingPolicy = false;
-
+
// Track capacities like
used-capcity/abs-used-capacity/capacity/abs-capacity,
// etc.
QueueCapacities queueCapacities;
@@ -129,12 +129,15 @@ public abstract class AbstractCSQueue
[hadoop] branch trunk updated: YARN-10458. Hive On Tez queries fails upon submission to dynamically created pools. (Peter Bacsko via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c47c9fd YARN-10458. Hive On Tez queries fails upon submission to
dynamically created pools. (Peter Bacsko via wangda)
c47c9fd is described below
commit c47c9fd65d835fc53c9eef59a05b97335b04e320
Author: Wangda Tan
AuthorDate: Fri Oct 30 09:49:48 2020 -0700
YARN-10458. Hive On Tez queries fails upon submission to dynamically
created pools. (Peter Bacsko via wangda)
Change-Id: I518dc925187ce55e9d35a37ba20878c0f4e37e5c
---
.../yarn/server/resourcemanager/RMAppManager.java | 4 +
.../scheduler/capacity/CapacityScheduler.java | 20 +
.../server/resourcemanager/TestAppManager.java | 94 +-
.../TestCapacitySchedulerAutoQueueCreation.java| 64 ++-
.../capacity/TestCapacitySchedulerQueueACLs.java | 42 ++
5 files changed, 219 insertions(+), 5 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 1e62f25..fe18d82 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -927,6 +927,10 @@ public class RMAppManager implements
EventHandler,
return usernameUsedForPlacement;
}
String queue = appPlacementContext.getQueue();
+ String parent = appPlacementContext.getParentQueue();
+ if (scheduler instanceof CapacityScheduler && parent != null) {
+queue = parent + "." + queue;
+ }
if (callerUGI != null && scheduler
.checkAccess(callerUGI, QueueACL.SUBMIT_APPLICATIONS, queue)) {
usernameUsedForPlacement = userNameFromAppTag;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index a632bfa..259cd5c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2290,6 +2290,21 @@ public class CapacityScheduler extends
public boolean checkAccess(UserGroupInformation callerUGI,
QueueACL acl, String queueName) {
CSQueue queue = getQueue(queueName);
+
+if (queueName.startsWith("root.")) {
+ // can only check proper ACLs if the path is fully qualified
+ while (queue == null) {
+int sepIndex = queueName.lastIndexOf(".");
+String parentName = queueName.substring(0, sepIndex);
+if (LOG.isDebugEnabled()) {
+ LOG.debug("Queue {} does not exist, checking parent {}",
+ queueName, parentName);
+}
+queueName = parentName;
+queue = queueManager.getQueue(queueName);
+ }
+}
+
if (queue == null) {
LOG.debug("ACL not found for queue access-type {} for queue {}",
acl, queueName);
@@ -3307,4 +3322,9 @@ public class CapacityScheduler extends
public void setActivitiesManagerEnabled(boolean enabled) {
this.activitiesManagerEnabled = enabled;
}
+
+ @VisibleForTesting
+ public void setQueueManager(CapacitySchedulerQueueManager qm) {
+this.queueManager = qm;
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index bc89917..e8b4105 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-
[hadoop] branch branch-3.1 updated: YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam Antal/Eric Yang via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 0ee7d09 YARN-9813. RM does not start on JDK11 when UIv2 is enabled.
(Adam Antal/Eric Yang via wangda)
0ee7d09 is described below
commit 0ee7d091387ac458b0744e8087d5dad95fd5855f
Author: Wangda Tan
AuthorDate: Fri Sep 6 19:18:13 2019 -0700
YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam
Antal/Eric Yang via wangda)
Change-Id: I18b8edc930b2efa0652f59c246931ad0d46827f3
(cherry picked from commit 34b82e6da0a471010cdae613ba39487889d79369)
---
.../org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index f54bd45..e3544ff 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -1178,8 +1178,7 @@ public class ResourceManager extends CompositeService
if (null == onDiskPath) {
String war = "hadoop-yarn-ui-" + VersionInfo.getVersion() + ".war";
-URLClassLoader cl = (URLClassLoader)
ClassLoader.getSystemClassLoader();
-URL url = cl.findResource(war);
+URL url = getClass().getClassLoader().getResource(war);
if (null == url) {
onDiskPath = getWebAppsPath("ui2");
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam Antal/Eric Yang via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 0e77347 YARN-9813. RM does not start on JDK11 when UIv2 is enabled.
(Adam Antal/Eric Yang via wangda)
0e77347 is described below
commit 0e77347972da9193b35b07a861b3d15d58c0ab47
Author: Wangda Tan
AuthorDate: Fri Sep 6 19:18:13 2019 -0700
YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam
Antal/Eric Yang via wangda)
Change-Id: I18b8edc930b2efa0652f59c246931ad0d46827f3
(cherry picked from commit 34b82e6da0a471010cdae613ba39487889d79369)
---
.../org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 41edfe4..65b0229 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -1202,8 +1202,7 @@ public class ResourceManager extends CompositeService
if (null == onDiskPath) {
String war = "hadoop-yarn-ui-" + VersionInfo.getVersion() + ".war";
-URLClassLoader cl = (URLClassLoader)
ClassLoader.getSystemClassLoader();
-URL url = cl.findResource(war);
+URL url = getClass().getClassLoader().getResource(war);
if (null == url) {
onDiskPath = getWebAppsPath("ui2");
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam Antal/Eric Yang via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 34b82e6 YARN-9813. RM does not start on JDK11 when UIv2 is enabled.
(Adam Antal/Eric Yang via wangda)
34b82e6 is described below
commit 34b82e6da0a471010cdae613ba39487889d79369
Author: Wangda Tan
AuthorDate: Fri Sep 6 19:18:13 2019 -0700
YARN-9813. RM does not start on JDK11 when UIv2 is enabled. (Adam
Antal/Eric Yang via wangda)
Change-Id: I18b8edc930b2efa0652f59c246931ad0d46827f3
---
.../org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 57be468..d939ac1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -1232,8 +1232,7 @@ public class ResourceManager extends CompositeService
if (null == onDiskPath) {
String war = "hadoop-yarn-ui-" + VersionInfo.getVersion() + ".war";
-URLClassLoader cl = (URLClassLoader)
ClassLoader.getSystemClassLoader();
-URL url = cl.findResource(war);
+URL url = getClass().getClassLoader().getResource(war);
if (null == url) {
onDiskPath = getWebAppsPath("ui2");
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-9319. Fix compilation issue of handling typedef an existing name by gcc compiler. (Zhankun Tang via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b17a260 YARN-9319. Fix compilation issue of handling typedef an
existing name by gcc compiler. (Zhankun Tang via wangda)
b17a260 is described below
commit b17a2602d1fcece0979feb1438a3e71099126bef
Author: Wangda Tan
AuthorDate: Thu Feb 21 11:17:32 2019 -0800
YARN-9319. Fix compilation issue of handling typedef an existing name by
gcc compiler. (Zhankun Tang via wangda)
Change-Id: Ie8293cc5b710ff355fb6a1ea7201be53ed0385bf
---
.../native/container-executor/impl/modules/devices/devices-module.c | 4 ++--
.../native/container-executor/impl/modules/devices/devices-module.h | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c
index 9df6662..4063bae 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c
@@ -82,7 +82,7 @@ cleanup:
}
static int internal_handle_devices_request(
-update_cgroups_parameters_function update_cgroups_parameters_func_p,
+update_cgroups_param_function update_cgroups_parameters_func_p,
char** deny_devices_number_tokens,
char** allow_devices_number_tokens,
const char* container_id) {
@@ -196,7 +196,7 @@ void reload_devices_configuration() {
* --allowed_devices 8:32,8:48,243:2 \
* --container_id container_x_y
*/
-int handle_devices_request(update_cgroups_parameters_function func,
+int handle_devices_request(update_cgroups_param_function func,
const char* module_name, int module_argc, char** module_argv) {
if (!cfg_section) {
reload_devices_configuration();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h
index c5d6785..552d070 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h
@@ -28,13 +28,13 @@
#define DEVICES_MODULE_SECTION_NAME "devices"
// For unit test stubbing
-typedef int (*update_cgroups_parameters_function)(const char*, const char*,
+typedef int (*update_cgroups_param_function)(const char*, const char*,
const char*, const char*);
/**
* Handle devices requests
*/
-int handle_devices_request(update_cgroups_parameters_function func,
+int handle_devices_request(update_cgroups_param_function func,
const char* module_name, int module_argc, char** module_argv);
/**
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] annotated tag rel/release-3.1.2 created (now b156fd9)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to annotated tag rel/release-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git. at b156fd9 (tag) tagging 1019dde65bcf12e05ef48ac71e84550d589e5d9a (commit) replaces release-3.1.2-RC0 by Wangda Tan on Tue Feb 5 13:26:33 2019 -0800 - Log - Hadoop 3.1.2 release -BEGIN PGP SIGNATURE- iQJGBAABCAAwFiEETImYU83aTkDGAhK1s/plPVcwDUUFAlxZ/4kSHHdhbmdkYUBh cGFjaGUub3JnAAoJELP6ZT1XMA1FqG8P/0/Mp2poBQEWtJVbL7VoQ3P2OMTyBvcy iyvrbjwa+QlxB/mFJLq9S05jPr8htwcukNi8AtbZgtxs1ovuBhtVu9bzbqbpSxlm aVAWbQ7pqu78/wSRKyTcS7JTikxZ2F4E9IEH18kZJcaGZzUHqDwq04phj5157BWk 5Xt7ilrM8W+4csA9K0BM1LUJPRfu9TzW/5HikOa0i/JWj/0NNzN9vvYjB0nLWRPj sp3d2FvDLQuJlkLnnLXIoGj9MIP4Qy2c3VhKjLeWHdIE6vMXG+xQqS/Kd5FFCIpy HtzdqeTk04WUCQ+ym0YBcmOtNcUn4yrR5fLab6rmyDBnMS7qZZ8dNp4KUBv8KBjt eiA5YgBfBDai0N4UXQCvwOD9g+KXmHsRXePrcvndCNB1k6g4oGM9uCHFxoKFu6/G eeaUR0KBOkGswD4hKX+BrvQnePs00IaI1m5+8LexIjY7HPuDbGBG0Co6y8mndJQq gvCNiCRV7FZlOWlxM+bUw4+G2lbOQXh5zrm2fK9boy51MuDF2zwpDD7mWFeldn8a kyAJx1W/P9LaEGgkDc2P19xIwxFsSWsoOt6RlrSxJeJsbWveAql6IBWCRGtgRk8R Vnkf9yoMVCmrMO1g/znkgHVafMP3xUU9T+xYMzKjKNW+IYA4moVjt9LtcnZA+4Ql i4UfrqzD0Jcn =Di3L -END PGP SIGNATURE- --- No new revisions were added by this update. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 02/04: YARN-8747. [UI2] YARN UI2 page loading failed due to js error under some time zone configuration. Contributed by collinma.
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit db87c107abbad9a3c6e34c2927e60f0d01a56079 Author: Akira Ajisaka AuthorDate: Wed Jan 16 14:35:07 2019 +0900 YARN-8747. [UI2] YARN UI2 page loading failed due to js error under some time zone configuration. Contributed by collinma. (cherry picked from commit 104ef5df36fd45f1c2d07a2f3d441263aa85e22e) (cherry picked from commit d07d275f942dcd9f65a8c867ffedd024c9307e0c) --- .../hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json index b7591a5..8fa9668 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json @@ -13,7 +13,7 @@ "qunit": "1.19.0", "jquery-ui": "1.11.4", "moment": "2.12.0", -"moment-timezone": "0.5.0", +"moment-timezone": "0.5.1", "more-js": "0.8.2", "bootstrap": "3.3.6", "d3": "~3.5.6", - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 01/04: YARN-9173. FairShare calculation broken for large values after YARN-8833. Contributed Wilfred Spiegelenburg.
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch branch-3.1.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit f6617d4c8bd8f5d226169e6870e703049eb456df
Author: Weiwei Yang
AuthorDate: Tue Jan 8 13:56:21 2019 +0800
YARN-9173. FairShare calculation broken for large values after YARN-8833.
Contributed Wilfred Spiegelenburg.
(cherry picked from commit 91e9c9f96eca01bbef7d22b4cff1ced3be1ac5e8)
---
.../scheduler/fair/policies/ComputeFairShares.java | 62
.../scheduler/fair/FakeSchedulable.java| 13 ++-
.../scheduler/fair/TestComputeFairShares.java | 104 ++---
3 files changed, 125 insertions(+), 54 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
index 3fe0c68..97bb4c5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
+import static java.lang.Math.addExact;
+
/**
* Contains logic for computing the fair shares. A {@link Schedulable}'s fair
* share is {@link Resource} it is entitled to, independent of the current
@@ -31,10 +33,13 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
* consumption lies at or below its fair share will never have its containers
* preempted.
*/
-public class ComputeFairShares {
+public final class ComputeFairShares {
private static final int COMPUTE_FAIR_SHARES_ITERATIONS = 25;
+ private ComputeFairShares() {
+ }
+
/**
* Compute fair share of the given schedulables.Fair share is an allocation
of
* shares considering only active schedulables ie schedulables which have
@@ -100,19 +105,20 @@ public class ComputeFairShares {
* all Schedulables are only given their minShare) and an upper bound
computed
* to be large enough that too many slots are given (by doubling R until we
* use more than totalResources resources). The helper method
- * resourceUsedWithWeightToResourceRatio computes the total resources used
with a
- * given value of R.
+ * resourceUsedWithWeightToResourceRatio computes the total resources used
+ * with a given value of R.
*
* The running time of this algorithm is linear in the number of
Schedulables,
- * because resourceUsedWithWeightToResourceRatio is linear-time and the
number of
- * iterations of binary search is a constant (dependent on desired
precision).
+ * because resourceUsedWithWeightToResourceRatio is linear-time and the
+ * number of iterations of binary search is a constant (dependent on desired
+ * precision).
*/
private static void computeSharesInternal(
Collection allSchedulables,
Resource totalResources, String type, boolean isSteadyShare) {
Collection schedulables = new ArrayList<>();
-int takenResources = handleFixedFairShares(
+long takenResources = handleFixedFairShares(
allSchedulables, schedulables, isSteadyShare, type);
if (schedulables.isEmpty()) {
@@ -121,12 +127,11 @@ public class ComputeFairShares {
// Find an upper bound on R that we can use in our binary search. We start
// at R = 1 and double it until we have either used all the resources or we
// have met all Schedulables' max shares.
-int totalMaxShare = 0;
+long totalMaxShare = 0;
for (Schedulable sched : schedulables) {
long maxShare = sched.getMaxShare().getResourceValue(type);
- totalMaxShare = (int) Math.min(maxShare + (long)totalMaxShare,
- Integer.MAX_VALUE);
- if (totalMaxShare == Integer.MAX_VALUE) {
+ totalMaxShare = safeAdd(maxShare, totalMaxShare);
+ if (totalMaxShare == Long.MAX_VALUE) {
break;
}
}
@@ -166,23 +171,24 @@ public class ComputeFairShares {
target = sched.getFairShare();
}
- target.setResourceValue(type, (long)computeShare(sched, right, type));
+ target.setResourceValue(type, computeShare(sched, right, type));
}
}
/**
* Compute the resources that would be used given a weight-to-resource
[hadoop] branch branch-3.1.2 updated (9f88af7 -> 211e2ca)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to branch branch-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 9f88af7 Revert "HDFS-14084. Need for more stats in DFSClient. Contributed by Pranay Singh." new f6617d4 YARN-9173. FairShare calculation broken for large values after YARN-8833. Contributed Wilfred Spiegelenburg. new db87c10 YARN-8747. [UI2] YARN UI2 page loading failed due to js error under some time zone configuration. Contributed by collinma. new dc4ca27 YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda) new 211e2ca YARN-9204. RM fails to start if absolute resource is specified for partition capacity in CS queues. Contributed by Jiandan Yang. The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../rmapp/attempt/RMAppAttemptImpl.java| 14 ++- .../capacity/CapacitySchedulerConfiguration.java | 24 - .../scheduler/fair/policies/ComputeFairShares.java | 62 .../rmapp/attempt/TestRMAppAttemptTransitions.java | 80 +++- .../scheduler/capacity/TestCapacityScheduler.java | 41 .../scheduler/fair/FakeSchedulable.java| 13 ++- .../scheduler/fair/TestComputeFairShares.java | 104 ++--- .../hadoop-yarn-ui/src/main/webapp/bower.json | 2 +- 8 files changed, 278 insertions(+), 62 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 04/04: YARN-9204. RM fails to start if absolute resource is specified for partition capacity in CS queues. Contributed by Jiandan Yang.
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch branch-3.1.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 211e2caef21fc386b798674fd0b7347d66fce837
Author: Weiwei Yang
AuthorDate: Mon Jan 21 16:54:14 2019 +0800
YARN-9204. RM fails to start if absolute resource is specified for
partition capacity in CS queues. Contributed by Jiandan Yang.
(cherry picked from commit abde1e1f58d5b699e4b8e460cff68e154738169b)
(cherry picked from commit 4edd883d4818dee3bcf2f8dea3e3d8ebf35d6605)
---
.../capacity/CapacitySchedulerConfiguration.java | 24 +++--
.../scheduler/capacity/TestCapacityScheduler.java | 41 ++
2 files changed, 62 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 547d131..1a300d2 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -471,6 +471,17 @@ public class CapacitySchedulerConfiguration extends
ReservationSchedulerConfigur
", capacity=" + capacity);
}
+ @VisibleForTesting
+ public void setCapacity(String queue, String absoluteResourceCapacity) {
+if (queue.equals("root")) {
+ throw new IllegalArgumentException(
+ "Cannot set capacity, root queue has a fixed capacity");
+}
+set(getQueuePrefix(queue) + CAPACITY, absoluteResourceCapacity);
+LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue)
++ ", capacity=" + absoluteResourceCapacity);
+ }
+
public float getNonLabeledQueueMaximumCapacity(String queue) {
String configuredCapacity = get(getQueuePrefix(queue) + MAXIMUM_CAPACITY);
boolean matcher = (configuredCapacity != null)
@@ -505,7 +516,13 @@ public class CapacitySchedulerConfiguration extends
ReservationSchedulerConfigur
public void setCapacityByLabel(String queue, String label, float capacity) {
setFloat(getNodeLabelPrefix(queue, label) + CAPACITY, capacity);
}
-
+
+ @VisibleForTesting
+ public void setCapacityByLabel(String queue, String label,
+ String absoluteResourceCapacity) {
+set(getNodeLabelPrefix(queue, label) + CAPACITY, absoluteResourceCapacity);
+ }
+
public void setMaximumCapacityByLabel(String queue, String label,
float capacity) {
setFloat(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY, capacity);
@@ -642,8 +659,9 @@ public class CapacitySchedulerConfiguration extends
ReservationSchedulerConfigur
private float internalGetLabeledQueueCapacity(String queue, String label,
String suffix,
float defaultValue) {
String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix;
-boolean matcher = (capacityPropertyName != null)
-&& RESOURCE_PATTERN.matcher(capacityPropertyName).find();
+String configuredCapacity = get(capacityPropertyName);
+boolean matcher = (configuredCapacity != null)
+&& RESOURCE_PATTERN.matcher(configuredCapacity).find();
if (matcher) {
// Return capacity in percentage as 0 for non-root queues and 100 for
// root.From AbstractCSQueue, absolute resource will be parsed and
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index db8836a..772e72e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -40,6 +40,7 @@ import java.util.Map;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CyclicBarrier;
+import com.google.common.collect.Sets;
import org.apache.
[hadoop] 03/04: YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch branch-3.1.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit dc4ca276771ff0381780f944e3539b27d7499d4f
Author: Wangda Tan
AuthorDate: Thu Jan 17 14:20:10 2019 -0800
YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and
NullPointerException happens in RM while shutdown a NM. (lujie via wangda)
Change-Id: I4359f59a73a278a941f4bb9d106dd38c9cb471fe
(cherry picked from commit 6d7eedfd28cc1712690db2f6ca8a281b0901ee28)
(cherry picked from commit fe7cb2d84ac160c5fed00640d85e2c5c4c6d2412)
(cherry picked from commit a685ffe9a9d5cf073faa2625d1125839fb57bae2)
---
.../rmapp/attempt/RMAppAttemptImpl.java| 14 +++-
.../rmapp/attempt/TestRMAppAttemptTransitions.java | 80 +-
2 files changed, 90 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index bb43693..feb690f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -437,9 +437,11 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
RMAppAttemptState.FAILED,
EnumSet.of(
RMAppAttemptEventType.LAUNCHED,
+ RMAppAttemptEventType.LAUNCH_FAILED,
RMAppAttemptEventType.EXPIRE,
RMAppAttemptEventType.KILL,
RMAppAttemptEventType.FAIL,
+ RMAppAttemptEventType.REGISTERED,
RMAppAttemptEventType.UNREGISTERED,
RMAppAttemptEventType.STATUS_UPDATE,
RMAppAttemptEventType.CONTAINER_ALLOCATED))
@@ -1203,10 +1205,16 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
}
// Set the masterContainer
- appAttempt.setMasterContainer(amContainerAllocation.getContainers()
- .get(0));
+ Container amContainer = amContainerAllocation.getContainers().get(0);
RMContainerImpl rmMasterContainer = (RMContainerImpl)appAttempt.scheduler
- .getRMContainer(appAttempt.getMasterContainer().getId());
+ .getRMContainer(amContainer.getId());
+ //while one NM is removed, the scheduler will clean the container,the
+ //following CONTAINER_FINISHED event will handle the cleaned container.
+ //so just return RMAppAttemptState.SCHEDULED
+ if (rmMasterContainer == null) {
+return RMAppAttemptState.SCHEDULED;
+ }
+ appAttempt.setMasterContainer(amContainer);
rmMasterContainer.setAMContainer(true);
// The node set in NMTokenSecrentManager is used for marking whether the
// NMToken has been issued for this node to the AM.
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 4a5c671..faecdb4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -986,7 +986,7 @@ public class TestRMAppAttemptTransitions {
public void testAttemptAddedAtFinalSaving() {
submitApplicationAttempt();
-// SUBNITED->FINAL_SAVING
+// SUBMITTED->FINAL_SAVING
applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
.getAppAttemptId(), RMAppAttemptEventType.KILL));
assertEquals(RMAppAttemptState.FINAL_SAVING,
@@ -999,6 +999,56 @@ public class TestRMAppAttemptTransitions {
applicationAttempt.getAppAttemptState());
}
+ @Test(timeout = 1)
+ public void testAttemptRegisteredAtFailed() {
+Container amContainer = allocateApplicationAttempt();
+launchApplicationAttempt(amContainer);
+
+//send CONTAINER_FINISHED event
+NodeId anyNodeId = NodeId.newInstance("
[hadoop] branch branch-3.1 updated: YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new a685ffe YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at
FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via
wangda)
a685ffe is described below
commit a685ffe9a9d5cf073faa2625d1125839fb57bae2
Author: Wangda Tan
AuthorDate: Thu Jan 17 14:20:10 2019 -0800
YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and
NullPointerException happens in RM while shutdown a NM. (lujie via wangda)
Change-Id: I4359f59a73a278a941f4bb9d106dd38c9cb471fe
(cherry picked from commit 6d7eedfd28cc1712690db2f6ca8a281b0901ee28)
(cherry picked from commit fe7cb2d84ac160c5fed00640d85e2c5c4c6d2412)
---
.../rmapp/attempt/RMAppAttemptImpl.java| 14 +++-
.../rmapp/attempt/TestRMAppAttemptTransitions.java | 80 +-
2 files changed, 90 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index bb43693..feb690f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -437,9 +437,11 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
RMAppAttemptState.FAILED,
EnumSet.of(
RMAppAttemptEventType.LAUNCHED,
+ RMAppAttemptEventType.LAUNCH_FAILED,
RMAppAttemptEventType.EXPIRE,
RMAppAttemptEventType.KILL,
RMAppAttemptEventType.FAIL,
+ RMAppAttemptEventType.REGISTERED,
RMAppAttemptEventType.UNREGISTERED,
RMAppAttemptEventType.STATUS_UPDATE,
RMAppAttemptEventType.CONTAINER_ALLOCATED))
@@ -1203,10 +1205,16 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
}
// Set the masterContainer
- appAttempt.setMasterContainer(amContainerAllocation.getContainers()
- .get(0));
+ Container amContainer = amContainerAllocation.getContainers().get(0);
RMContainerImpl rmMasterContainer = (RMContainerImpl)appAttempt.scheduler
- .getRMContainer(appAttempt.getMasterContainer().getId());
+ .getRMContainer(amContainer.getId());
+ //while one NM is removed, the scheduler will clean the container,the
+ //following CONTAINER_FINISHED event will handle the cleaned container.
+ //so just return RMAppAttemptState.SCHEDULED
+ if (rmMasterContainer == null) {
+return RMAppAttemptState.SCHEDULED;
+ }
+ appAttempt.setMasterContainer(amContainer);
rmMasterContainer.setAMContainer(true);
// The node set in NMTokenSecrentManager is used for marking whether the
// NMToken has been issued for this node to the AM.
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 4a5c671..faecdb4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -986,7 +986,7 @@ public class TestRMAppAttemptTransitions {
public void testAttemptAddedAtFinalSaving() {
submitApplicationAttempt();
-// SUBNITED->FINAL_SAVING
+// SUBMITTED->FINAL_SAVING
applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
.getAppAttemptId(), RMAppAttemptEventType.KILL));
assertEquals(RMAppAttemptState.FINAL_SAVING,
@@ -999,6 +999,56 @@ public class TestRMAppAttemptTransitions {
applicationAttempt.getAppAttemptState());
}
+ @Test(timeout = 1)
+ public void testAttemptRegisteredAt
[hadoop] branch branch-3.2 updated: YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new fe7cb2d YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at
FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via
wangda)
fe7cb2d is described below
commit fe7cb2d84ac160c5fed00640d85e2c5c4c6d2412
Author: Wangda Tan
AuthorDate: Thu Jan 17 14:20:10 2019 -0800
YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and
NullPointerException happens in RM while shutdown a NM. (lujie via wangda)
Change-Id: I4359f59a73a278a941f4bb9d106dd38c9cb471fe
(cherry picked from commit 6d7eedfd28cc1712690db2f6ca8a281b0901ee28)
---
.../rmapp/attempt/RMAppAttemptImpl.java| 14 +++-
.../rmapp/attempt/TestRMAppAttemptTransitions.java | 80 +-
2 files changed, 90 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 3ec9c49..03039da 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -437,9 +437,11 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
RMAppAttemptState.FAILED,
EnumSet.of(
RMAppAttemptEventType.LAUNCHED,
+ RMAppAttemptEventType.LAUNCH_FAILED,
RMAppAttemptEventType.EXPIRE,
RMAppAttemptEventType.KILL,
RMAppAttemptEventType.FAIL,
+ RMAppAttemptEventType.REGISTERED,
RMAppAttemptEventType.UNREGISTERED,
RMAppAttemptEventType.STATUS_UPDATE,
RMAppAttemptEventType.CONTAINER_ALLOCATED))
@@ -1203,10 +1205,16 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
}
// Set the masterContainer
- appAttempt.setMasterContainer(amContainerAllocation.getContainers()
- .get(0));
+ Container amContainer = amContainerAllocation.getContainers().get(0);
RMContainerImpl rmMasterContainer = (RMContainerImpl)appAttempt.scheduler
- .getRMContainer(appAttempt.getMasterContainer().getId());
+ .getRMContainer(amContainer.getId());
+ //while one NM is removed, the scheduler will clean the container,the
+ //following CONTAINER_FINISHED event will handle the cleaned container.
+ //so just return RMAppAttemptState.SCHEDULED
+ if (rmMasterContainer == null) {
+return RMAppAttemptState.SCHEDULED;
+ }
+ appAttempt.setMasterContainer(amContainer);
rmMasterContainer.setAMContainer(true);
// The node set in NMTokenSecrentManager is used for marking whether the
// NMToken has been issued for this node to the AM.
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 4a5c671..faecdb4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -986,7 +986,7 @@ public class TestRMAppAttemptTransitions {
public void testAttemptAddedAtFinalSaving() {
submitApplicationAttempt();
-// SUBNITED->FINAL_SAVING
+// SUBMITTED->FINAL_SAVING
applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
.getAppAttemptId(), RMAppAttemptEventType.KILL));
assertEquals(RMAppAttemptState.FINAL_SAVING,
@@ -999,6 +999,56 @@ public class TestRMAppAttemptTransitions {
applicationAttempt.getAppAttemptState());
}
+ @Test(timeout = 1)
+ public void testAttemptRegisteredAtFailed() {
+Container amContainer = allocateApplicationA
[hadoop] branch trunk updated: YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6d7eedf YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at
FAILED, and NullPointerException happens in RM while shutdown a NM. (lujie via
wangda)
6d7eedf is described below
commit 6d7eedfd28cc1712690db2f6ca8a281b0901ee28
Author: Wangda Tan
AuthorDate: Thu Jan 17 14:20:10 2019 -0800
YARN-9194. Invalid event: REGISTERED and LAUNCH_FAILED at FAILED, and
NullPointerException happens in RM while shutdown a NM. (lujie via wangda)
Change-Id: I4359f59a73a278a941f4bb9d106dd38c9cb471fe
---
.../rmapp/attempt/RMAppAttemptImpl.java| 14 +++-
.../rmapp/attempt/TestRMAppAttemptTransitions.java | 80 +-
2 files changed, 90 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 3ec9c49..03039da 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -437,9 +437,11 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
RMAppAttemptState.FAILED,
EnumSet.of(
RMAppAttemptEventType.LAUNCHED,
+ RMAppAttemptEventType.LAUNCH_FAILED,
RMAppAttemptEventType.EXPIRE,
RMAppAttemptEventType.KILL,
RMAppAttemptEventType.FAIL,
+ RMAppAttemptEventType.REGISTERED,
RMAppAttemptEventType.UNREGISTERED,
RMAppAttemptEventType.STATUS_UPDATE,
RMAppAttemptEventType.CONTAINER_ALLOCATED))
@@ -1203,10 +1205,16 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
}
// Set the masterContainer
- appAttempt.setMasterContainer(amContainerAllocation.getContainers()
- .get(0));
+ Container amContainer = amContainerAllocation.getContainers().get(0);
RMContainerImpl rmMasterContainer = (RMContainerImpl)appAttempt.scheduler
- .getRMContainer(appAttempt.getMasterContainer().getId());
+ .getRMContainer(amContainer.getId());
+ //while one NM is removed, the scheduler will clean the container,the
+ //following CONTAINER_FINISHED event will handle the cleaned container.
+ //so just return RMAppAttemptState.SCHEDULED
+ if (rmMasterContainer == null) {
+return RMAppAttemptState.SCHEDULED;
+ }
+ appAttempt.setMasterContainer(amContainer);
rmMasterContainer.setAMContainer(true);
// The node set in NMTokenSecrentManager is used for marking whether the
// NMToken has been issued for this node to the AM.
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 4a5c671..faecdb4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -986,7 +986,7 @@ public class TestRMAppAttemptTransitions {
public void testAttemptAddedAtFinalSaving() {
submitApplicationAttempt();
-// SUBNITED->FINAL_SAVING
+// SUBMITTED->FINAL_SAVING
applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
.getAppAttemptId(), RMAppAttemptEventType.KILL));
assertEquals(RMAppAttemptState.FINAL_SAVING,
@@ -999,6 +999,56 @@ public class TestRMAppAttemptTransitions {
applicationAttempt.getAppAttemptState());
}
+ @Test(timeout = 1)
+ public void testAttemptRegisteredAtFailed() {
+Container amContainer = allocateApplicationAttempt();
+launchApplicationAttempt(amContainer);
+
+//send CONTAINER_FINISHED event
+NodeId any
[hadoop] annotated tag release-3.1.2-RC0 created (now ac9a81a)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to annotated tag release-3.1.2-RC0 in repository https://gitbox.apache.org/repos/asf/hadoop.git. at ac9a81a (tag) tagging 9f88af79029606b29a5d0ff9bea661a2a83a1dfb (commit) by Wangda Tan on Fri Jan 11 16:16:59 2019 -0800 - Log - Release candidate - 3.1.2-RC0 -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEETImYU83aTkDGAhK1s/plPVcwDUUFAlw5MfwACgkQs/plPVcw DUWUwg/9HZdjZV9i5teFuavrSOyguJ3ZJYqLzngAVVqln/66F/TxV511ScpYZInF nPmh2kFIGLuVMN4/FzAui04at0eonMroCrZ17wcu0zQh5evhy7LtS87ghPukhFOX RiAduDGauRHbSLuImERMIXH39S1sW48sRr1nBuJjFZjPJDkOVcerR0oYnt8ZX3P9 vy2Tb+B7xpq+8bkUxUJn0ysiUzqy0l8+UVA4iYJ9o3yWJ6WOTgbnqGU4e7s4F9Kv EdtBakiyvx8f0gS25LFs6LmS90pLtlxEQb+BY60cXxAWVfWf+sIHK2/Jsjq5A2LP +1lpR6gRtXmwTbXulWdqbpaZO2+k0bcR0O/3LfKAEQvgGA5DI8/5K+iiurXvLDLD CT5ZVUhMw9R//ZsPveKDPlm+wmDgqzK13SvfZZ/XSS0LouQ9/419oxenHR6Yp0WS dpycHqHFPa3ufxCSgN207p8YwsZihdxmGILCbCuzDQwWKncJTs6eZjYRuYAAUKCS ByWY1wHN6sDvhpvAWzQwwK9bOghH0yMJj+Shmm8prr4s0Hwh9hCi8S4BfIv3hz5v 3l9i6ohZA1Kwf9inRetLTkhFStIWLTUcFE78ycKJ9vc+VBK2y2RhCpSuzsHrj2in wCRLEjGvYzjB0U0NTkKVpPtu8aVPin6rcdCotk28cneapsCzUuk= =0WUH -END PGP SIGNATURE- --- No new revisions were added by this update. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.1.2 updated: YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch branch-3.1.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1.2 by this push:
new e5c4ebb YARN-8822. Nvidia-docker v2 support for YARN GPU feature.
(Charo Zhang via wangda)
e5c4ebb is described below
commit e5c4ebb8c94cfede5a437dab75519f5c7f93f7f0
Author: Wangda Tan
AuthorDate: Mon Jan 7 12:30:30 2019 -0800
YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via
wangda)
Change-Id: I41626a7b6f097d218d84e8497dd70b4b6d8f
(cherry picked from commit 3570713a692e9985b51f534de50ce6d613f3b052)
---
.../hadoop-yarn/conf/container-executor.cfg| 1 +
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 3 +
.../linux/runtime/docker/DockerRunCommand.java | 5 +
.../gpu/GpuDockerCommandPluginFactory.java | 4 +
.../gpu/NvidiaDockerV2CommandPlugin.java | 111 ++
.../container-executor/impl/utils/docker-util.c| 20
.../container-executor/impl/utils/docker-util.h| 3 +-
.../test/utils/test_docker_util.cc | 62 ++
.../linux/runtime/docker/TestDockerRunCommand.java | 5 +-
.../gpu/TestNvidiaDockerV2CommandPlugin.java | 130 +
.../src/site/markdown/DockerContainers.md | 1 +
.../src/site/markdown/UsingGpus.md | 9 +-
12 files changed, 351 insertions(+), 3 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
index d19874f..4df53df 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
+++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
@@ -16,6 +16,7 @@ feature.tc.enabled=false
# docker.privileged-containers.enabled=false
# docker.allowed.volume-drivers=## comma seperated list of allowed
volume-drivers
# docker.no-new-privileges.enabled=## enable/disable the no-new-privileges
flag for docker run. Set to "true" to enable, disabled by default
+# docker.allowed.runtimes=## comma seperated runtimes that can be used.
# The configs below deal with settings for FPGA resource
#[fpga]
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index a5fb723..366b261 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1597,6 +1597,9 @@ public class YarnConfiguration extends Configuration {
public static final String NVIDIA_DOCKER_V1 = "nvidia-docker-v1";
@Private
+ public static final String NVIDIA_DOCKER_V2 = "nvidia-docker-v2";
+
+ @Private
public static final String DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL =
NVIDIA_DOCKER_V1;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index 241163d..2864e98 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -165,6 +165,11 @@ public class DockerRunCommand extends DockerCommand {
return this;
}
+ public DockerRunCommand addRuntime(String runtime) {
+super.addCommandArguments("runtime", runtime);
+return this;
+ }
+
public DockerRunCommand groupAdd(String[] groups) {
super.addCommandArguments("group-add", String.join(",", groups));
return this;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java
index db4589a..051afd6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-se
[hadoop] branch branch-3.1 updated: YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 3570713 YARN-8822. Nvidia-docker v2 support for YARN GPU feature.
(Charo Zhang via wangda)
3570713 is described below
commit 3570713a692e9985b51f534de50ce6d613f3b052
Author: Wangda Tan
AuthorDate: Mon Jan 7 12:30:30 2019 -0800
YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via
wangda)
Change-Id: I41626a7b6f097d218d84e8497dd70b4b6d8f
---
.../hadoop-yarn/conf/container-executor.cfg| 1 +
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 3 +
.../linux/runtime/docker/DockerRunCommand.java | 5 +
.../gpu/GpuDockerCommandPluginFactory.java | 4 +
.../gpu/NvidiaDockerV2CommandPlugin.java | 111 ++
.../container-executor/impl/utils/docker-util.c| 20
.../container-executor/impl/utils/docker-util.h| 3 +-
.../test/utils/test_docker_util.cc | 62 ++
.../linux/runtime/docker/TestDockerRunCommand.java | 5 +-
.../gpu/TestNvidiaDockerV2CommandPlugin.java | 130 +
.../src/site/markdown/DockerContainers.md | 1 +
.../src/site/markdown/UsingGpus.md | 9 +-
12 files changed, 351 insertions(+), 3 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
index d19874f..4df53df 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
+++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
@@ -16,6 +16,7 @@ feature.tc.enabled=false
# docker.privileged-containers.enabled=false
# docker.allowed.volume-drivers=## comma seperated list of allowed
volume-drivers
# docker.no-new-privileges.enabled=## enable/disable the no-new-privileges
flag for docker run. Set to "true" to enable, disabled by default
+# docker.allowed.runtimes=## comma seperated runtimes that can be used.
# The configs below deal with settings for FPGA resource
#[fpga]
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index a5fb723..366b261 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1597,6 +1597,9 @@ public class YarnConfiguration extends Configuration {
public static final String NVIDIA_DOCKER_V1 = "nvidia-docker-v1";
@Private
+ public static final String NVIDIA_DOCKER_V2 = "nvidia-docker-v2";
+
+ @Private
public static final String DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL =
NVIDIA_DOCKER_V1;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index 241163d..2864e98 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -165,6 +165,11 @@ public class DockerRunCommand extends DockerCommand {
return this;
}
+ public DockerRunCommand addRuntime(String runtime) {
+super.addCommandArguments("runtime", runtime);
+return this;
+ }
+
public DockerRunCommand groupAdd(String[] groups) {
super.addCommandArguments("group-add", String.join(",", groups));
return this;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java
index db4589a..051afd6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containerm
[hadoop] branch branch-3.2 updated: YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 1dc2b49 YARN-8822. Nvidia-docker v2 support for YARN GPU feature.
(Charo Zhang via wangda)
1dc2b49 is described below
commit 1dc2b49bfd74b61302cb2d771ce33fe939a393f7
Author: Wangda Tan
AuthorDate: Mon Jan 7 12:21:33 2019 -0800
YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via
wangda)
Change-Id: Ib8044307a4241f6b1b7b9b8266b9256f39b16384
---
.../hadoop-yarn/conf/container-executor.cfg| 1 +
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 3 +
.../linux/runtime/docker/DockerRunCommand.java | 5 +
.../gpu/GpuDockerCommandPluginFactory.java | 4 +
.../gpu/NvidiaDockerV2CommandPlugin.java | 111 ++
.../container-executor/impl/utils/docker-util.c| 20
.../container-executor/impl/utils/docker-util.h| 3 +-
.../test/utils/test_docker_util.cc | 62 ++
.../linux/runtime/docker/TestDockerRunCommand.java | 5 +-
.../gpu/TestNvidiaDockerV2CommandPlugin.java | 130 +
.../src/site/markdown/DockerContainers.md | 1 +
.../src/site/markdown/UsingGpus.md | 9 +-
12 files changed, 351 insertions(+), 3 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
index d19874f..4df53df 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
+++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
@@ -16,6 +16,7 @@ feature.tc.enabled=false
# docker.privileged-containers.enabled=false
# docker.allowed.volume-drivers=## comma seperated list of allowed
volume-drivers
# docker.no-new-privileges.enabled=## enable/disable the no-new-privileges
flag for docker run. Set to "true" to enable, disabled by default
+# docker.allowed.runtimes=## comma seperated runtimes that can be used.
# The configs below deal with settings for FPGA resource
#[fpga]
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index c5abec7..5f15931 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1636,6 +1636,9 @@ public class YarnConfiguration extends Configuration {
public static final String NVIDIA_DOCKER_V1 = "nvidia-docker-v1";
@Private
+ public static final String NVIDIA_DOCKER_V2 = "nvidia-docker-v2";
+
+ @Private
public static final String DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL =
NVIDIA_DOCKER_V1;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index 241163d..2864e98 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -165,6 +165,11 @@ public class DockerRunCommand extends DockerCommand {
return this;
}
+ public DockerRunCommand addRuntime(String runtime) {
+super.addCommandArguments("runtime", runtime);
+return this;
+ }
+
public DockerRunCommand groupAdd(String[] groups) {
super.addCommandArguments("group-add", String.join(",", groups));
return this;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java
index db4589a..051afd6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containerm
[hadoop] branch trunk updated: YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 0a01d49 YARN-8822. Nvidia-docker v2 support for YARN GPU feature.
(Charo Zhang via wangda)
0a01d49 is described below
commit 0a01d499175569ffac9d3f31d980820ecac7e60b
Author: Wangda Tan
AuthorDate: Mon Jan 7 12:07:26 2019 -0800
YARN-8822. Nvidia-docker v2 support for YARN GPU feature. (Charo Zhang via
wangda)
Change-Id: Id8af27134d3286a7a10d85eda9be25df9689d0e7
---
.../hadoop-yarn/conf/container-executor.cfg| 1 +
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 3 +
.../linux/runtime/docker/DockerRunCommand.java | 5 +
.../gpu/GpuDockerCommandPluginFactory.java | 4 +
.../gpu/NvidiaDockerV2CommandPlugin.java | 111 ++
.../container-executor/impl/utils/docker-util.c| 20
.../container-executor/impl/utils/docker-util.h| 3 +-
.../test/utils/test_docker_util.cc | 62 ++
.../linux/runtime/docker/TestDockerRunCommand.java | 5 +-
.../gpu/TestNvidiaDockerV2CommandPlugin.java | 130 +
.../src/site/markdown/DockerContainers.md | 1 +
.../src/site/markdown/UsingGpus.md | 9 +-
12 files changed, 351 insertions(+), 3 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
index d19874f..4df53df 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
+++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
@@ -16,6 +16,7 @@ feature.tc.enabled=false
# docker.privileged-containers.enabled=false
# docker.allowed.volume-drivers=## comma seperated list of allowed
volume-drivers
# docker.no-new-privileges.enabled=## enable/disable the no-new-privileges
flag for docker run. Set to "true" to enable, disabled by default
+# docker.allowed.runtimes=## comma seperated runtimes that can be used.
# The configs below deal with settings for FPGA resource
#[fpga]
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 08f7f1a..c29707c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1665,6 +1665,9 @@ public class YarnConfiguration extends Configuration {
public static final String NVIDIA_DOCKER_V1 = "nvidia-docker-v1";
@Private
+ public static final String NVIDIA_DOCKER_V2 = "nvidia-docker-v2";
+
+ @Private
public static final String DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL =
NVIDIA_DOCKER_V1;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index f4f3a9c..b0603a3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -165,6 +165,11 @@ public class DockerRunCommand extends DockerCommand {
return this;
}
+ public DockerRunCommand addRuntime(String runtime) {
+super.addCommandArguments("runtime", runtime);
+return this;
+ }
+
public DockerRunCommand groupAdd(String[] groups) {
super.addCommandArguments("group-add", String.join(",", groups));
return this;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDockerCommandPluginFactory.java
index db4589a..051afd6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourc
[hadoop] 01/01: Preparing for 3.1.2 release
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit 733d03485b019af02a4cbd3b60569ed2153fef4d Author: Wangda Tan AuthorDate: Mon Jan 7 10:09:01 2019 -0800 Preparing for 3.1.2 release Change-Id: I46544065b0352024c8b6a23e96a8e10170c925ba --- hadoop-assemblies/pom.xml | 4 ++-- hadoop-build-tools/pom.xml| 2 +- hadoop-client-modules/hadoop-client-api/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-check-invariants/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-integration-tests/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-minicluster/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-runtime/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client/pom.xml | 4 ++-- hadoop-client-modules/pom.xml | 2 +- hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml | 4 ++-- hadoop-cloud-storage-project/pom.xml | 4 ++-- hadoop-common-project/hadoop-annotations/pom.xml | 4 ++-- hadoop-common-project/hadoop-auth-examples/pom.xml| 4 ++-- hadoop-common-project/hadoop-auth/pom.xml | 4 ++-- hadoop-common-project/hadoop-common/pom.xml | 4 ++-- hadoop-common-project/hadoop-kms/pom.xml | 4 ++-- hadoop-common-project/hadoop-minikdc/pom.xml | 4 ++-- hadoop-common-project/hadoop-nfs/pom.xml | 4 ++-- hadoop-common-project/pom.xml | 4 ++-- hadoop-dist/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml| 4 ++-- hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml| 4 ++-- hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 ++-- hadoop-hdfs-project/pom.xml | 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml | 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml | 4 ++-- .../hadoop-mapreduce-client-hs-plugins/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml | 4 ++-- .../hadoop-mapreduce-client-nativetask/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml | 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml | 4 ++-- hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 4 ++-- hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml| 4 ++-- hadoop-mapreduce-project/pom.xml | 4 ++-- hadoop-maven-plugins/pom.xml | 2 +- hadoop-minicluster/pom.xml| 4 ++-- hadoop-project-dist/pom.xml | 4 ++-- hadoop-project/pom.xml| 4 ++-- hadoop-tools/hadoop-aliyun/pom.xml| 2 +- hadoop-tools/hadoop-archive-logs/pom.xml | 4 ++-- hadoop-tools/hadoop-archives/pom.xml | 4 ++-- hadoop-tools/hadoop-aws/pom.xml | 4 ++-- hadoop-tools/hadoop-azure-datalake/pom.xml| 2 +- hadoop-tools/hadoop-azure/pom.xml | 2 +- hadoop-tools/hadoop-datajoin/pom.xml | 4 ++-- hadoop-tools/hadoop-distcp/pom.xml| 4 ++-- hadoop-tools/hadoop-extras/pom.xml| 4 ++-- hadoop-tools/hadoop-fs2img/pom.xml| 4 ++-- hadoop-tools/hadoop-gridmix/pom.xml | 4 ++-- hadoop-tools/hadoop-kafka/pom.xml | 4 ++-- hadoop-tools/hadoop-openstack/pom.xml | 4 ++-- hadoop-tools/hadoop-pipes/pom.xml
[hadoop] branch branch-3.1.2 created (now 733d034)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to branch branch-3.1.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git. at 733d034 Preparing for 3.1.2 release This branch includes the following new commits: new 733d034 Preparing for 3.1.2 release The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.1 updated: Preparing for 3.1.3 development
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.1 by this push: new 31ea2f7 Preparing for 3.1.3 development 31ea2f7 is described below commit 31ea2f780631d40478e3ce9d6fef70832081f720 Author: Wangda Tan AuthorDate: Mon Jan 7 10:04:58 2019 -0800 Preparing for 3.1.3 development Change-Id: I3c3d3ee47dc4fef239127b4452ff14676fa26e3d --- hadoop-assemblies/pom.xml | 4 ++-- hadoop-build-tools/pom.xml| 2 +- hadoop-client-modules/hadoop-client-api/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-check-invariants/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-integration-tests/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-minicluster/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client-runtime/pom.xml | 4 ++-- hadoop-client-modules/hadoop-client/pom.xml | 4 ++-- hadoop-client-modules/pom.xml | 2 +- hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml | 4 ++-- hadoop-cloud-storage-project/pom.xml | 4 ++-- hadoop-common-project/hadoop-annotations/pom.xml | 4 ++-- hadoop-common-project/hadoop-auth-examples/pom.xml| 4 ++-- hadoop-common-project/hadoop-auth/pom.xml | 4 ++-- hadoop-common-project/hadoop-common/pom.xml | 4 ++-- hadoop-common-project/hadoop-kms/pom.xml | 4 ++-- hadoop-common-project/hadoop-minikdc/pom.xml | 4 ++-- hadoop-common-project/hadoop-nfs/pom.xml | 4 ++-- hadoop-common-project/pom.xml | 4 ++-- hadoop-dist/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml| 4 ++-- hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml| 4 ++-- hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 ++-- hadoop-hdfs-project/pom.xml | 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml | 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml | 4 ++-- .../hadoop-mapreduce-client-hs-plugins/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml | 4 ++-- .../hadoop-mapreduce-client-nativetask/pom.xml| 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml | 4 ++-- .../hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml | 4 ++-- hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 4 ++-- hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml| 4 ++-- hadoop-mapreduce-project/pom.xml | 4 ++-- hadoop-maven-plugins/pom.xml | 2 +- hadoop-minicluster/pom.xml| 4 ++-- hadoop-project-dist/pom.xml | 4 ++-- hadoop-project/pom.xml| 4 ++-- hadoop-tools/hadoop-aliyun/pom.xml| 2 +- hadoop-tools/hadoop-archive-logs/pom.xml | 4 ++-- hadoop-tools/hadoop-archives/pom.xml | 4 ++-- hadoop-tools/hadoop-aws/pom.xml | 4 ++-- hadoop-tools/hadoop-azure-datalake/pom.xml| 2 +- hadoop-tools/hadoop-azure/pom.xml | 2 +- hadoop-tools/hadoop-datajoin/pom.xml | 4 ++-- hadoop-tools/hadoop-distcp/pom.xml| 4 ++-- hadoop-tools/hadoop-extras/pom.xml| 4 ++-- hadoop-tools/hadoop-fs2img/pom.xml| 4 ++-- hadoop-tools/hadoop-gridmix/pom.xml | 4 ++-- hadoop-tools/hadoop-kafka/pom.xml
[hadoop] branch branch-3.2 updated (71bee05 -> cffe5c1)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 71bee05 Revert "HADOOP-15759. AliyunOSS: Update oss-sdk version to 3.0.0. Contributed by Jinhu Wu." new 1ad1ab5 YARN-9141. [Submarine] JobStatus outputs with system UTC clock, not local clock. (Zac Zhou via wangda) new cffe5c1 YARN-9160. [Submarine] Document 'PYTHONPATH' environment variable setting when using -localization options. (Zhankun Tang via wangda) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../hadoop/yarn/submarine/common/api/JobStatus.java| 4 ++-- .../src/site/markdown/QuickStart.md| 18 ++ 2 files changed, 20 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 02/02: YARN-9160. [Submarine] Document 'PYTHONPATH' environment variable setting when using -localization options. (Zhankun Tang via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git commit cffe5c1ba089cb6a720f479a40a8e300127caa44 Author: Wangda Tan AuthorDate: Sun Jan 6 10:25:35 2019 -0800 YARN-9160. [Submarine] Document 'PYTHONPATH' environment variable setting when using -localization options. (Zhankun Tang via wangda) Change-Id: I115b6710eb11501e21ab3e9e7cd1a87f322e5cf8 (cherry picked from commit 2c02aa6ec259128934cc5468cf66104a624d88a7) --- .../src/site/markdown/QuickStart.md| 18 ++ 1 file changed, 18 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md index 8e7a956..21db6bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md @@ -87,6 +87,24 @@ usage: job run -localization "/user/yarn/mydir3:/opt/mydir3" -localization "./mydir1:." ``` + + Notes: +When using `localization` option to make a collection of dependency Python +scripts available to entry python script in the container, you may also need to + set `PYTHONPATH` environment variable as below to avoid module import error +reported from `entry_script.py`. + +``` +... job run + # the entry point + --localization entry_script.py:/entry_script.py + # the dependency Python scripts of the entry point + --localization other_scripts_dir:/other_scripts_dir + # the PYTHONPATH env to make dependency available to entry script + --env PYTHONPATH="/other_scripts_dir" + --worker_launch_cmd "python /entry_script.py ..." +``` + ### Submarine Configuration For submarine internal configuration, please create a `submarine.xml` which should be placed under `$HADOOP_CONF_DIR`. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 01/02: YARN-9141. [Submarine] JobStatus outputs with system UTC clock, not local clock. (Zac Zhou via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 1ad1ab5d7a2a292039ccb06ae659f691e9ef63aa
Author: Wangda Tan
AuthorDate: Sun Jan 6 10:24:29 2019 -0800
YARN-9141. [Submarine] JobStatus outputs with system UTC clock, not local
clock. (Zac Zhou via wangda)
Change-Id: Ib2b17cf7860ddc9188f06498086de3326bc09519
(cherry picked from commit 8f004fe4e0d479e26d0063427566edc1606bced5)
---
.../java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java
index 6e390f3..0ba9de9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.yarn.submarine.common.api;
import java.io.PrintStream;
-import java.time.Instant;
+import java.time.LocalDateTime;
import java.util.List;
/**
@@ -34,7 +34,7 @@ public class JobStatus {
public void nicePrint(PrintStream out) {
out.println(
"Job Name=" + this.jobName + ", status=" + state.name() + " time="
-+ Instant.now());
++ LocalDateTime.now());
if (JobState.isFinal(this.state)) {
return;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 01/02: YARN-9141. [Submarine] JobStatus outputs with system UTC clock, not local clock. (Zac Zhou via wangda)
This is an automated email from the ASF dual-hosted git repository.
wangda pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 8f004fe4e0d479e26d0063427566edc1606bced5
Author: Wangda Tan
AuthorDate: Sun Jan 6 10:24:29 2019 -0800
YARN-9141. [Submarine] JobStatus outputs with system UTC clock, not local
clock. (Zac Zhou via wangda)
Change-Id: Ib2b17cf7860ddc9188f06498086de3326bc09519
---
.../java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java
index 6e390f3..0ba9de9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/api/JobStatus.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.yarn.submarine.common.api;
import java.io.PrintStream;
-import java.time.Instant;
+import java.time.LocalDateTime;
import java.util.List;
/**
@@ -34,7 +34,7 @@ public class JobStatus {
public void nicePrint(PrintStream out) {
out.println(
"Job Name=" + this.jobName + ", status=" + state.name() + " time="
-+ Instant.now());
++ LocalDateTime.now());
if (JobState.isFinal(this.state)) {
return;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 02/02: YARN-9160. [Submarine] Document 'PYTHONPATH' environment variable setting when using -localization options. (Zhankun Tang via wangda)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git commit 2c02aa6ec259128934cc5468cf66104a624d88a7 Author: Wangda Tan AuthorDate: Sun Jan 6 10:25:35 2019 -0800 YARN-9160. [Submarine] Document 'PYTHONPATH' environment variable setting when using -localization options. (Zhankun Tang via wangda) Change-Id: I115b6710eb11501e21ab3e9e7cd1a87f322e5cf8 --- .../src/site/markdown/QuickStart.md| 18 ++ 1 file changed, 18 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md index 8e7a956..21db6bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/QuickStart.md @@ -87,6 +87,24 @@ usage: job run -localization "/user/yarn/mydir3:/opt/mydir3" -localization "./mydir1:." ``` + + Notes: +When using `localization` option to make a collection of dependency Python +scripts available to entry python script in the container, you may also need to + set `PYTHONPATH` environment variable as below to avoid module import error +reported from `entry_script.py`. + +``` +... job run + # the entry point + --localization entry_script.py:/entry_script.py + # the dependency Python scripts of the entry point + --localization other_scripts_dir:/other_scripts_dir + # the PYTHONPATH env to make dependency available to entry script + --env PYTHONPATH="/other_scripts_dir" + --worker_launch_cmd "python /entry_script.py ..." +``` + ### Submarine Configuration For submarine internal configuration, please create a `submarine.xml` which should be placed under `$HADOOP_CONF_DIR`. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (650b9cb -> 2c02aa6)
This is an automated email from the ASF dual-hosted git repository. wangda pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 650b9cb YARN-9178. TestRMAdminCli#testHelp is failing in trunk. Contributed by Abhishek Modi. new 8f004fe YARN-9141. [Submarine] JobStatus outputs with system UTC clock, not local clock. (Zac Zhou via wangda) new 2c02aa6 YARN-9160. [Submarine] Document 'PYTHONPATH' environment variable setting when using -localization options. (Zhankun Tang via wangda) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../hadoop/yarn/submarine/common/api/JobStatus.java| 4 ++-- .../src/site/markdown/QuickStart.md| 18 ++ 2 files changed, 20 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[3/3] hadoop git commit: YARN-9112. [Submarine] Support polling applicationId when it's not ready in cluster. (Zhankun Tang via wangda)
YARN-9112. [Submarine] Support polling applicationId when it's not ready in
cluster. (Zhankun Tang via wangda)
Change-Id: I73d73f3d631b28fb9866faa56571839b13824a97
(cherry picked from commit 9fba6cc2471db8df9ca6da7b9df378e3072dfeb2)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29e4e5f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29e4e5f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29e4e5f6
Branch: refs/heads/branch-3.2
Commit: 29e4e5f62c3cd3a2bbd3160d8721f518301765e0
Parents: ef0f835
Author: Wangda Tan
Authored: Wed Dec 12 11:44:58 2018 -0800
Committer: Wangda Tan
Committed: Wed Dec 12 11:52:16 2018 -0800
--
.../yarnservice/YarnServiceJobSubmitter.java| 24
1 file changed, 20 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29e4e5f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
index 8cc9794..b92438d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
@@ -866,10 +866,26 @@ public class YarnServiceJobSubmitter implements
JobSubmitter {
}
String appStatus=appAdminClient.getStatusString(serviceSpec.getName());
-Service app=ServiceApiUtil.jsonSerDeser.fromJson(appStatus);
-if(app.getId() == null) {
- throw new YarnException("Can't get application id for Service " +
- serviceSpec.getName());
+Service app = ServiceApiUtil.jsonSerDeser.fromJson(appStatus);
+
+// Retry multiple times if applicationId is null
+int maxRetryTimes = 30;
+int count = 0;
+while (app.getId() == null && count < maxRetryTimes) {
+ LOG.info("Waiting for application Id. AppStatusString=\n {}", appStatus);
+ try {
+Thread.sleep(1000);
+ } catch (InterruptedException e) {
+throw new IOException(e);
+ }
+ appStatus = appAdminClient.getStatusString(serviceSpec.getName());
+ app = ServiceApiUtil.jsonSerDeser.fromJson(appStatus);
+ count++;
+}
+// Retry timeout
+if (app.getId() == null) {
+ throw new YarnException(
+ "Can't get application id for Service " + serviceSpec.getName());
}
ApplicationId appid = ApplicationId.fromString(app.getId());
appAdminClient.stop();
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/3] hadoop git commit: YARN-8714. [Submarine] Support files/tarballs to be localized for a training job. (Zhankun Tang via wangda)
YARN-8714. [Submarine] Support files/tarballs to be localized for a training
job. (Zhankun Tang via wangda)
Change-Id: I845131273e52a9d81dbc813ea6d4af06b205e334
(cherry picked from commit c771fe6e10bd3f8533a01026de3c33db2a983cf4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef0f8355
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef0f8355
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef0f8355
Branch: refs/heads/branch-3.2
Commit: ef0f8355372652a55bf1680ddccab0012f55a283
Parents: bbe1ddd
Author: Wangda Tan
Authored: Wed Dec 12 11:43:23 2018 -0800
Committer: Wangda Tan
Committed: Wed Dec 12 11:52:07 2018 -0800
--
.../yarn/submarine/client/cli/CliConstants.java | 1 +
.../yarn/submarine/client/cli/RunJobCli.java| 18 +
.../client/cli/param/Localization.java | 133
.../client/cli/param/RunJobParameters.java | 15 +
.../client/cli/param/package-info.java | 19 +
.../common/conf/SubmarineConfiguration.java | 15 +
.../fs/DefaultRemoteDirectoryManager.java | 70 +-
.../common/fs/RemoteDirectoryManager.java | 18 +-
.../common/FSBasedSubmarineStorageImpl.java | 8 +-
.../yarnservice/YarnServiceJobSubmitter.java| 259 +-
.../src/site/markdown/QuickStart.md | 26 +
.../yarnservice/TestYarnServiceRunJobCli.java | 795 ++-
.../submarine/common/MockClientContext.java | 8 +-
.../common/fs/MockRemoteDirectoryManager.java | 90 ++-
14 files changed, 1433 insertions(+), 42 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef0f8355/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
index 2d7a472..da4253b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
@@ -52,6 +52,7 @@ public class CliConstants {
public static final String QUICKLINK = "quicklink";
public static final String TENSORBOARD_DOCKER_IMAGE =
"tensorboard_docker_image";
+ public static final String LOCALIZATION = "localization";
public static final String KEYTAB = "keytab";
public static final String PRINCIPAL = "principal";
public static final String DISTRIBUTE_KEYTAB = "distribute_keytab";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef0f8355/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
index fc57be9..b764d6d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
@@ -125,6 +125,24 @@ public class RunJobCli extends AbstractCli {
+ "if want to link to first worker's 7070 port, and text of quicklink "
+ "is Notebook_UI, user need to specify --quicklink "
+ "Notebook_UI=https://master-0:7070;);
+options.addOption(CliConstants.LOCALIZATION, true, "Specify"
++ " localization to make remote/local file/directory available to"
++ " all container(Docker)."
++ " Argument format is \"RemoteUri:LocalFilePath[:rw] \" (ro"
++ " permission is not supported yet)"
++ " The RemoteUri can be a file or directory in local or"
++ " HDFS or s3 or abfs or http .etc."
++ " The LocalFilePath can be absolute o
[1/3] hadoop git commit: YARN-8714. [Submarine] Support files/tarballs to be localized for a training job. (Zhankun Tang via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.2 bbe1ddd30 -> 29e4e5f62
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef0f8355/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java
index b637036..4334293 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java
@@ -19,7 +19,9 @@
package org.apache.hadoop.yarn.submarine.common.fs;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import java.io.File;
@@ -29,6 +31,7 @@ public class MockRemoteDirectoryManager implements
RemoteDirectoryManager {
private File jobsParentDir = null;
private File modelParentDir = null;
+ private File jobDir = null;
@Override
public Path getJobStagingArea(String jobName, boolean create)
throws IOException {
@@ -41,10 +44,11 @@ public class MockRemoteDirectoryManager implements
RemoteDirectoryManager {
}
}
-File jobDir = new File(jobsParentDir.getAbsolutePath(), jobName);
+this.jobDir = new File(jobsParentDir.getAbsolutePath(), jobName);
if (create && !jobDir.exists()) {
if (!jobDir.mkdirs()) {
-throw new IOException("Failed to mkdirs for " +
jobDir.getAbsolutePath());
+throw new IOException("Failed to mkdirs for "
++ jobDir.getAbsolutePath());
}
}
return new Path(jobDir.getAbsolutePath());
@@ -57,7 +61,8 @@ public class MockRemoteDirectoryManager implements
RemoteDirectoryManager {
}
@Override
- public Path getModelDir(String modelName, boolean create) throws IOException
{
+ public Path getModelDir(String modelName, boolean create)
+ throws IOException {
if (modelParentDir == null && create) {
modelParentDir = new File(
"target/_models_" + System.currentTimeMillis());
@@ -70,19 +75,94 @@ public class MockRemoteDirectoryManager implements
RemoteDirectoryManager {
File modelDir = new File(modelParentDir.getAbsolutePath(), modelName);
if (create) {
if (!modelDir.exists() && !modelDir.mkdirs()) {
-throw new IOException("Failed to mkdirs for " +
modelDir.getAbsolutePath());
+throw new IOException("Failed to mkdirs for "
++ modelDir.getAbsolutePath());
}
}
return new Path(modelDir.getAbsolutePath());
}
@Override
- public FileSystem getFileSystem() throws IOException {
+ public FileSystem getDefaultFileSystem() throws IOException {
return FileSystem.getLocal(new Configuration());
}
@Override
+ public FileSystem getFileSystemByUri(String uri) throws IOException {
+return getDefaultFileSystem();
+ }
+
+ @Override
public Path getUserRootFolder() throws IOException {
return new Path("s3://generated_root_dir");
}
+
+ @Override
+ public boolean isDir(String uri) throws IOException {
+return getDefaultFileSystem().getFileStatus(
+new Path(convertToStagingPath(uri))).isDirectory();
+
+ }
+
+ @Override
+ public boolean isRemote(String uri) throws IOException {
+String scheme = new Path(uri).toUri().getScheme();
+if (null == scheme) {
+ return false;
+}
+return !scheme.startsWith("file://");
+ }
+
+ private String convertToStagingPath(String uri) throws IOException {
+String ret = uri;
+if (isRemote(uri)) {
+ String dirName = new Path(uri).getName();
+ ret = this.jobDir.getAbsolutePath()
+ + "/" + dirName;
+}
+return ret;
+ }
+
+ /**
+ * We use staging dir as mock HDFS dir.
+ * */
+ @Override
+ public boolean copyRemoteToLocal(String remoteUri, String localUri)
+ throws IOException {
+// mock the copy from HDFS into a local copy
+Path remoteToLocalDir = new Path(convertToStagingPath(remoteUri));
+File old = new File(convertToStagingPath(localUri));
+if (old.isDirectory() && old.exists()) {
+ if (!FileUtil.fullyDelete(old)) {
+throw new IOException("Cannot delete temp dir:"
++ old.getAbsolutePath());
+ }
+}
+return FileUtil.copy(getDefaultFileSystem(), remoteToLocalDir,
[6/6] hadoop git commit: YARN-8885. [DevicePlugin] Support NM APIs to query device resource allocation. (Zhankun Tang via wangda)
YARN-8885. [DevicePlugin] Support NM APIs to query device resource allocation.
(Zhankun Tang via wangda)
Change-Id: I2a9870709b512af1ac6c09c9701d0b3c0791ff32
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37eb919c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37eb919c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37eb919c
Branch: refs/heads/trunk
Commit: 37eb919c59d10955213f5b8230e5687887f1f9c3
Parents: 9fba6cc
Author: Wangda Tan
Authored: Wed Dec 12 11:45:47 2018 -0800
Committer: Wangda Tan
Committed: Wed Dec 12 11:45:47 2018 -0800
--
.../deviceframework/AssignedDevice.java | 78
.../deviceframework/DevicePluginAdapter.java| 18 -
.../webapp/dao/NMDeviceResourceInfo.java| 56 ++
.../TestDevicePluginAdapter.java| 67 +
4 files changed, 218 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37eb919c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/AssignedDevice.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/AssignedDevice.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/AssignedDevice.java
new file mode 100644
index 000..cad95ab
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/AssignedDevice.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package
org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Device wrapper class used for NM REST API.
+ * */
+public class AssignedDevice implements Serializable, Comparable {
+
+ private static final long serialVersionUID = -544285507952217366L;
+
+ private Device device;
+ private String containerId;
+
+ public AssignedDevice(ContainerId cId, Device dev) {
+this.device = dev;
+this.containerId = cId.toString();
+ }
+
+ public Device getDevice() {
+return device;
+ }
+
+ public String getContainerId() {
+return containerId;
+ }
+
+ @Override
+ public int compareTo(Object o) {
+if (o == null || !(o instanceof AssignedDevice)) {
+ return -1;
+}
+AssignedDevice other = (AssignedDevice) o;
+int result = getDevice().compareTo(other.getDevice());
+if (0 != result) {
+ return result;
+}
+return getContainerId().compareTo(other.getContainerId());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+if (o == null || !(o instanceof AssignedDevice)) {
+ return false;
+}
+AssignedDevice other = (AssignedDevice) o;
+return getDevice().equals(other.getDevice())
+&& getContainerId().equals(other.getContainerId());
+ }
+
+ @Override
+ public int hashCode() {
+return Objects.hash(getDevice(), getContainerId());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37eb919c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DevicePluginAdapter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-ya
[4/6] hadoop git commit: YARN-9015. [DevicePlugin] Add an interface for device plugin to provide customized scheduler. (Zhankun Tang via wangda)
YARN-9015. [DevicePlugin] Add an interface for device plugin to provide
customized scheduler. (Zhankun Tang via wangda)
Change-Id: Ib2e4ae47a6f29bb3082c1f8520cf5a52ca720979
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61bdcb7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61bdcb7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61bdcb7b
Branch: refs/heads/trunk
Commit: 61bdcb7b2bc3057a8fe2b9dfb2b8154a585eab87
Parents: c771fe6
Author: Wangda Tan
Authored: Wed Dec 12 11:44:22 2018 -0800
Committer: Wangda Tan
Committed: Wed Dec 12 11:44:22 2018 -0800
--
.../api/deviceplugin/DevicePluginScheduler.java | 38 ++
.../resourceplugin/ResourcePluginManager.java | 29 +++-
.../deviceframework/DeviceMappingManager.java | 73 +---
.../deviceframework/DevicePluginAdapter.java| 2 +-
.../TestResourcePluginManager.java | 28
.../deviceframework/FakeTestDevicePlugin1.java | 18 -
.../deviceframework/FakeTestDevicePlugin5.java | 56 +++
.../TestDevicePluginAdapter.java| 70 ++-
8 files changed, 300 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61bdcb7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePluginScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePluginScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePluginScheduler.java
new file mode 100644
index 000..80ffdf7
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePluginScheduler.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin;
+
+import java.util.Set;
+
+/**
+ * An optional interface to implement if custom device scheduling is needed.
+ * If this is not implemented, the device framework will do scheduling.
+ * */
+public interface DevicePluginScheduler {
+ /**
+ * Called when allocating devices. The framework will do all device book
+ * keeping and fail recovery. So this hook could be stateless and only do
+ * scheduling based on available devices passed in. It could be
+ * invoked multiple times by the framework.
+ * @param availableDevices Devices allowed to be chosen from.
+ * @param count Number of device to be allocated.
+ * @return A set of {@link Device} allocated
+ * */
+ Set allocateDevices(Set availableDevices, int count);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61bdcb7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
index bd47cfe..0bc241d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
+++
b/hadoop-yarn-projec
[3/6] hadoop git commit: YARN-8714. [Submarine] Support files/tarballs to be localized for a training job. (Zhankun Tang via wangda)
YARN-8714. [Submarine] Support files/tarballs to be localized for a training
job. (Zhankun Tang via wangda)
Change-Id: I845131273e52a9d81dbc813ea6d4af06b205e334
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c771fe6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c771fe6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c771fe6e
Branch: refs/heads/trunk
Commit: c771fe6e10bd3f8533a01026de3c33db2a983cf4
Parents: 8a53611
Author: Wangda Tan
Authored: Wed Dec 12 11:43:23 2018 -0800
Committer: Wangda Tan
Committed: Wed Dec 12 11:43:23 2018 -0800
--
.../yarn/submarine/client/cli/CliConstants.java | 1 +
.../yarn/submarine/client/cli/RunJobCli.java| 18 +
.../client/cli/param/Localization.java | 133
.../client/cli/param/RunJobParameters.java | 15 +
.../client/cli/param/package-info.java | 19 +
.../common/conf/SubmarineConfiguration.java | 15 +
.../fs/DefaultRemoteDirectoryManager.java | 70 +-
.../common/fs/RemoteDirectoryManager.java | 18 +-
.../common/FSBasedSubmarineStorageImpl.java | 8 +-
.../yarnservice/YarnServiceJobSubmitter.java| 259 +-
.../src/site/markdown/QuickStart.md | 26 +
.../yarnservice/TestYarnServiceRunJobCli.java | 795 ++-
.../submarine/common/MockClientContext.java | 8 +-
.../common/fs/MockRemoteDirectoryManager.java | 90 ++-
14 files changed, 1433 insertions(+), 42 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c771fe6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
index 2d7a472..da4253b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
@@ -52,6 +52,7 @@ public class CliConstants {
public static final String QUICKLINK = "quicklink";
public static final String TENSORBOARD_DOCKER_IMAGE =
"tensorboard_docker_image";
+ public static final String LOCALIZATION = "localization";
public static final String KEYTAB = "keytab";
public static final String PRINCIPAL = "principal";
public static final String DISTRIBUTE_KEYTAB = "distribute_keytab";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c771fe6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
index fc57be9..b764d6d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
@@ -125,6 +125,24 @@ public class RunJobCli extends AbstractCli {
+ "if want to link to first worker's 7070 port, and text of quicklink "
+ "is Notebook_UI, user need to specify --quicklink "
+ "Notebook_UI=https://master-0:7070;);
+options.addOption(CliConstants.LOCALIZATION, true, "Specify"
++ " localization to make remote/local file/directory available to"
++ " all container(Docker)."
++ " Argument format is \"RemoteUri:LocalFilePath[:rw] \" (ro"
++ " permission is not supported yet)"
++ " The RemoteUri can be a file or directory in local or"
++ " HDFS or s3 or abfs or http .etc."
++ " The LocalFilePath can be absolute or relative."
++ " If it's a relative path, it'll be"
+
[1/6] hadoop git commit: YARN-9078. [Submarine] Clean up the code of CliUtils#parseResourcesString. (Zhankun Tang via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 561b61cfb -> 37eb919c5
YARN-9078. [Submarine] Clean up the code of CliUtils#parseResourcesString.
(Zhankun Tang via wangda)
Change-Id: I21ac5757b6115d55ec3157ba25db1b65bc85a37b
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a536111
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a536111
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a536111
Branch: refs/heads/trunk
Commit: 8a5361116e6158b28b185ba903ece4aa454344c8
Parents: 561b61c
Author: Wangda Tan
Authored: Wed Dec 12 11:42:44 2018 -0800
Committer: Wangda Tan
Committed: Wed Dec 12 11:42:44 2018 -0800
--
.../yarn/submarine/client/cli/CliUtils.java | 23 +++-
1 file changed, 8 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a536111/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
index 588a9e6..f3eee7c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
@@ -74,19 +74,10 @@ public class CliUtils {
return newCli;
}
- // TODO, this duplicated to Client of distributed shell, should cleanup
private static Map parseResourcesString(String resourcesStr) {
Map resources = new HashMap<>();
-
-// Ignore the grouping "[]"
-if (resourcesStr.startsWith("[")) {
- resourcesStr = resourcesStr.substring(1);
-}
-if (resourcesStr.endsWith("]")) {
- resourcesStr = resourcesStr.substring(0, resourcesStr.length() - 1);
-}
-
-for (String resource : resourcesStr.trim().split(",")) {
+String[] pairs = resourcesStr.trim().split(",");
+for (String resource : pairs) {
resource = resource.trim();
if (!resource.matches(RES_PATTERN)) {
throw new IllegalArgumentException("\"" + resource + "\" is not a "
@@ -97,8 +88,9 @@ public class CliUtils {
String key = splits[0], value = splits[1];
String units = ResourceUtils.getUnits(value);
- String valueWithoutUnit = value.substring(0, value.length() -
units.length()).trim();
- Long resourceValue = Long.valueOf(valueWithoutUnit);
+ String valueWithoutUnit = value.substring(0,
+ value.length()- units.length()).trim();
+ long resourceValue = Long.parseLong(valueWithoutUnit);
// Convert commandline unit to standard YARN unit.
if (units.equals("M") || units.equals("m")) {
@@ -107,7 +99,7 @@ public class CliUtils {
units = "Gi";
} else if (units.isEmpty()) {
// do nothing;
- } else{
+ } else {
throw new IllegalArgumentException("Acceptable units are M/G or
empty");
}
@@ -121,7 +113,8 @@ public class CliUtils {
if (key.equals("memory")) {
key = ResourceInformation.MEMORY_URI;
-resourceValue = UnitsConversionUtil.convert(units, "Mi",
resourceValue);
+resourceValue = UnitsConversionUtil.convert(units, "Mi",
+resourceValue);
}
// special handle gpu
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[5/6] hadoop git commit: YARN-9112. [Submarine] Support polling applicationId when it's not ready in cluster. (Zhankun Tang via wangda)
YARN-9112. [Submarine] Support polling applicationId when it's not ready in
cluster. (Zhankun Tang via wangda)
Change-Id: I73d73f3d631b28fb9866faa56571839b13824a97
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9fba6cc2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9fba6cc2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9fba6cc2
Branch: refs/heads/trunk
Commit: 9fba6cc2471db8df9ca6da7b9df378e3072dfeb2
Parents: 61bdcb7
Author: Wangda Tan
Authored: Wed Dec 12 11:44:58 2018 -0800
Committer: Wangda Tan
Committed: Wed Dec 12 11:44:58 2018 -0800
--
.../yarnservice/YarnServiceJobSubmitter.java| 24
1 file changed, 20 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fba6cc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
index 496ba7c..06d33b2 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
@@ -865,10 +865,26 @@ public class YarnServiceJobSubmitter implements
JobSubmitter {
}
String appStatus=appAdminClient.getStatusString(serviceSpec.getName());
-Service app=ServiceApiUtil.jsonSerDeser.fromJson(appStatus);
-if(app.getId() == null) {
- throw new YarnException("Can't get application id for Service " +
- serviceSpec.getName());
+Service app = ServiceApiUtil.jsonSerDeser.fromJson(appStatus);
+
+// Retry multiple times if applicationId is null
+int maxRetryTimes = 30;
+int count = 0;
+while (app.getId() == null && count < maxRetryTimes) {
+ LOG.info("Waiting for application Id. AppStatusString=\n {}", appStatus);
+ try {
+Thread.sleep(1000);
+ } catch (InterruptedException e) {
+throw new IOException(e);
+ }
+ appStatus = appAdminClient.getStatusString(serviceSpec.getName());
+ app = ServiceApiUtil.jsonSerDeser.fromJson(appStatus);
+ count++;
+}
+// Retry timeout
+if (app.getId() == null) {
+ throw new YarnException(
+ "Can't get application id for Service " + serviceSpec.getName());
}
ApplicationId appid = ApplicationId.fromString(app.getId());
appAdminClient.stop();
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)" This reverts commit 920401b2966c2dcd7cd1b3dfd7a764f08412323f. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/829b530a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/829b530a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/829b530a Branch: refs/heads/branch-3.2.0 Commit: 829b530a082307f9c275f4e9e4a1ee35aefbf4d6 Parents: 542b224 Author: Wangda Tan Authored: Tue Dec 4 14:14:41 2018 -0800 Committer: Wangda Tan Committed: Tue Dec 4 14:14:41 2018 -0800 -- .../resources/assemblies/hadoop-yarn-dist.xml | 8 - .../installation/install.conf | 74 .../installation/install.sh | 116 - .../package/calico/calico-node.service | 50 --- .../installation/package/calico/calicoctl.cfg | 22 - .../installation/package/docker/daemon.json | 23 - .../installation/package/docker/docker.service | 35 -- .../installation/package/etcd/etcd.service | 40 -- .../package/hadoop/container-executor.cfg | 41 -- .../installation/package/submarine/submarine.sh | 25 -- .../installation/scripts/calico.sh | 224 -- .../installation/scripts/docker.sh | 166 --- .../installation/scripts/download-server.sh | 42 -- .../installation/scripts/environment.sh | 213 - .../installation/scripts/etcd.sh| 152 --- .../installation/scripts/hadoop.sh | 117 - .../installation/scripts/menu.sh| 444 --- .../installation/scripts/nvidia-docker.sh | 99 - .../installation/scripts/nvidia.sh | 120 - .../installation/scripts/submarine.sh | 38 -- .../installation/scripts/utils.sh | 123 - 21 files changed, 2172 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/829b530a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 9c401e8..a2ea08c 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -66,14 +66,6 @@ 0755 - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation - /share/hadoop/yarn/submarine-installer - -**/* - - 0755 - - hadoop-yarn/conf etc/hadoop http://git-wip-us.apache.org/repos/asf/hadoop/blob/829b530a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf deleted file mode 100644 index 82dcf61..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# install config # - -# DNS -LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf -YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address - -# etcd hosts list -ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) - -# docker registry ip:port -DOCKER_REGISTRY="10.120.196.232:5000" - -# Start the http download service on the specified server, -# Will download all the dependencies in the http server, -# Run the install script on other servers. -# Automatically download dependencies from http, -# Solve the problem that all servers are slow to
[1/2] hadoop git commit: Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
Repository: hadoop
Updated Branches:
refs/heads/branch-3.2.0 542b224d1 -> 829b530a0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/829b530a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
deleted file mode 100644
index 7b3c2a9..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description check install user
-## @audience public
-## @stabilitystable
-function check_install_user()
-{
- if [[ $(id -u) -ne 0 ]];then
-echo "This script must be run with a ROOT user!"
-exit # don't call exit_install()
- fi
-}
-
-## @description exit install
-## @audience public
-## @stabilitystable
-function exit_install()
-{
- echo "Exit the installation!" | tee -a $LOG
- exit $1
-}
-
-## @description Check if the IP address format is correct
-## @audience public
-## @stabilitystable
-function valid_ip()
-{
- local ip=$1
- local stat=1
-
- if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then
-OIFS=$IFS
-IFS='.'
-ip=($ip)
-IFS=$OIFS
-
-if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]}
-le 255 ]]; then
- stat=$?
-fi
- fi
-
- return $stat
-}
-
-## @description Check if the configuration file configuration is correct
-## @audience public
-## @stabilitystable
-function check_install_conf()
-{
- echo "Check if the configuration file configuration is correct ..." | tee -a
$LOG
-
- # check etcd conf
- hostCount=${#ETCD_HOSTS[@]}
- if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2
-echo "Number of nodes = [$hostCount], must be configured to be greater
than or equal to 3 servers! " | tee -a $LOG
-exit_install
- fi
- for ip in ${ETCD_HOSTS[@]}
- do
-if ! valid_ip $ip; then
- echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG
- exit_install
-fi
- done
- echo "Check if the configuration file configuration is correct [ Done ]" |
tee -a $LOG
-}
-
-## @description index by EtcdHosts list
-## @audience public
-## @stabilitystable
-function indexByEtcdHosts() {
- index=0
- while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do
-if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then
- echo $index
- return
-fi
-let "index++"
- done
- echo ""
-}
-
-## @description get local IP
-## @audience public
-## @stabilitystable
-function getLocalIP()
-{
- local _ip _myip _line _nl=$'\n'
- while IFS=$': \t' read -a _line ;do
- [ -z "${_line%inet}" ] &&
- _ip=${_line[${#_line[1]}>4?1:2]} &&
- [ "${_ip#127.0.0.1}" ] && _myip=$_ip
-done< <(LANG=C /sbin/ifconfig)
- printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip
-}
-
-## @description get ip list
-## @audience public
-## @stabilitystable
-function get_ip_list()
-{
- array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[
\t]*//g' | cut -d ' ' -f2)
-
- for ip in ${array[@]}
- do
-LOCAL_HOST_IP_LIST+=(${ip})
- done
-}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)" This reverts commit 920401b2966c2dcd7cd1b3dfd7a764f08412323f. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/559cb99b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/559cb99b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/559cb99b Branch: refs/heads/branch-3.2 Commit: 559cb99bf69f9af4b101bd3f0c1d6d5f99f02a6f Parents: 05e41dd Author: Wangda Tan Authored: Tue Dec 4 14:13:39 2018 -0800 Committer: Wangda Tan Committed: Tue Dec 4 14:13:39 2018 -0800 -- .../resources/assemblies/hadoop-yarn-dist.xml | 8 - .../installation/install.conf | 74 .../installation/install.sh | 116 - .../package/calico/calico-node.service | 50 --- .../installation/package/calico/calicoctl.cfg | 22 - .../installation/package/docker/daemon.json | 23 - .../installation/package/docker/docker.service | 35 -- .../installation/package/etcd/etcd.service | 40 -- .../package/hadoop/container-executor.cfg | 41 -- .../installation/package/submarine/submarine.sh | 25 -- .../installation/scripts/calico.sh | 224 -- .../installation/scripts/docker.sh | 166 --- .../installation/scripts/download-server.sh | 42 -- .../installation/scripts/environment.sh | 213 - .../installation/scripts/etcd.sh| 152 --- .../installation/scripts/hadoop.sh | 117 - .../installation/scripts/menu.sh| 444 --- .../installation/scripts/nvidia-docker.sh | 99 - .../installation/scripts/nvidia.sh | 120 - .../installation/scripts/submarine.sh | 38 -- .../installation/scripts/utils.sh | 123 - 21 files changed, 2172 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/559cb99b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 9c401e8..a2ea08c 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -66,14 +66,6 @@ 0755 - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation - /share/hadoop/yarn/submarine-installer - -**/* - - 0755 - - hadoop-yarn/conf etc/hadoop http://git-wip-us.apache.org/repos/asf/hadoop/blob/559cb99b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf deleted file mode 100644 index 82dcf61..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# install config # - -# DNS -LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf -YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address - -# etcd hosts list -ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) - -# docker registry ip:port -DOCKER_REGISTRY="10.120.196.232:5000" - -# Start the http download service on the specified server, -# Will download all the dependencies in the http server, -# Run the install script on other servers. -# Automatically download dependencies from http, -# Solve the problem that all servers are slow to
[1/2] hadoop git commit: Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
Repository: hadoop
Updated Branches:
refs/heads/branch-3.2 05e41dd4f -> 559cb99bf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/559cb99b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
deleted file mode 100644
index 7b3c2a9..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description check install user
-## @audience public
-## @stabilitystable
-function check_install_user()
-{
- if [[ $(id -u) -ne 0 ]];then
-echo "This script must be run with a ROOT user!"
-exit # don't call exit_install()
- fi
-}
-
-## @description exit install
-## @audience public
-## @stabilitystable
-function exit_install()
-{
- echo "Exit the installation!" | tee -a $LOG
- exit $1
-}
-
-## @description Check if the IP address format is correct
-## @audience public
-## @stabilitystable
-function valid_ip()
-{
- local ip=$1
- local stat=1
-
- if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then
-OIFS=$IFS
-IFS='.'
-ip=($ip)
-IFS=$OIFS
-
-if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]}
-le 255 ]]; then
- stat=$?
-fi
- fi
-
- return $stat
-}
-
-## @description Check if the configuration file configuration is correct
-## @audience public
-## @stabilitystable
-function check_install_conf()
-{
- echo "Check if the configuration file configuration is correct ..." | tee -a
$LOG
-
- # check etcd conf
- hostCount=${#ETCD_HOSTS[@]}
- if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2
-echo "Number of nodes = [$hostCount], must be configured to be greater
than or equal to 3 servers! " | tee -a $LOG
-exit_install
- fi
- for ip in ${ETCD_HOSTS[@]}
- do
-if ! valid_ip $ip; then
- echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG
- exit_install
-fi
- done
- echo "Check if the configuration file configuration is correct [ Done ]" |
tee -a $LOG
-}
-
-## @description index by EtcdHosts list
-## @audience public
-## @stabilitystable
-function indexByEtcdHosts() {
- index=0
- while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do
-if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then
- echo $index
- return
-fi
-let "index++"
- done
- echo ""
-}
-
-## @description get local IP
-## @audience public
-## @stabilitystable
-function getLocalIP()
-{
- local _ip _myip _line _nl=$'\n'
- while IFS=$': \t' read -a _line ;do
- [ -z "${_line%inet}" ] &&
- _ip=${_line[${#_line[1]}>4?1:2]} &&
- [ "${_ip#127.0.0.1}" ] && _myip=$_ip
-done< <(LANG=C /sbin/ifconfig)
- printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip
-}
-
-## @description get ip list
-## @audience public
-## @stabilitystable
-function get_ip_list()
-{
- array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[
\t]*//g' | cut -d ' ' -f2)
-
- for ip in ${array[@]}
- do
-LOCAL_HOST_IP_LIST+=(${ip})
- done
-}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)" This reverts commit 46d6e0016610ced51a76189daeb3ad0e3dbbf94c. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/228156cf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/228156cf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/228156cf Branch: refs/heads/trunk Commit: 228156cfd1b474988bc4fedfbf7edddc87db41e3 Parents: 9287ab3 Author: Wangda Tan Authored: Tue Dec 4 14:13:06 2018 -0800 Committer: Wangda Tan Committed: Tue Dec 4 14:13:06 2018 -0800 -- .../resources/assemblies/hadoop-yarn-dist.xml | 8 - .../installation/install.conf | 74 .../installation/install.sh | 116 - .../package/calico/calico-node.service | 50 --- .../installation/package/calico/calicoctl.cfg | 22 - .../installation/package/docker/daemon.json | 23 - .../installation/package/docker/docker.service | 35 -- .../installation/package/etcd/etcd.service | 40 -- .../package/hadoop/container-executor.cfg | 41 -- .../installation/package/submarine/submarine.sh | 25 -- .../installation/scripts/calico.sh | 224 -- .../installation/scripts/docker.sh | 166 --- .../installation/scripts/download-server.sh | 42 -- .../installation/scripts/environment.sh | 213 - .../installation/scripts/etcd.sh| 152 --- .../installation/scripts/hadoop.sh | 117 - .../installation/scripts/menu.sh| 444 --- .../installation/scripts/nvidia-docker.sh | 99 - .../installation/scripts/nvidia.sh | 120 - .../installation/scripts/submarine.sh | 38 -- .../installation/scripts/utils.sh | 123 - 21 files changed, 2172 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 69ab66a..4055acb 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -66,14 +66,6 @@ 0755 - hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation - /share/hadoop/yarn/submarine-installer - -**/* - - 0755 - - hadoop-yarn/conf etc/hadoop http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf deleted file mode 100644 index 82dcf61..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# install config # - -# DNS -LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf -YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address - -# etcd hosts list -ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) - -# docker registry ip:port -DOCKER_REGISTRY="10.120.196.232:5000" - -# Start the http download service on the specified server, -# Will download all the dependencies in the http server, -# Run the install script on other servers. -# Automatically download dependencies from http, -# Solve the problem that all servers are slow to
[1/2] hadoop git commit: Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
Repository: hadoop
Updated Branches:
refs/heads/trunk 9287ab364 -> 228156cfd
http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
deleted file mode 100644
index 7b3c2a9..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description check install user
-## @audience public
-## @stabilitystable
-function check_install_user()
-{
- if [[ $(id -u) -ne 0 ]];then
-echo "This script must be run with a ROOT user!"
-exit # don't call exit_install()
- fi
-}
-
-## @description exit install
-## @audience public
-## @stabilitystable
-function exit_install()
-{
- echo "Exit the installation!" | tee -a $LOG
- exit $1
-}
-
-## @description Check if the IP address format is correct
-## @audience public
-## @stabilitystable
-function valid_ip()
-{
- local ip=$1
- local stat=1
-
- if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then
-OIFS=$IFS
-IFS='.'
-ip=($ip)
-IFS=$OIFS
-
-if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]}
-le 255 ]]; then
- stat=$?
-fi
- fi
-
- return $stat
-}
-
-## @description Check if the configuration file configuration is correct
-## @audience public
-## @stabilitystable
-function check_install_conf()
-{
- echo "Check if the configuration file configuration is correct ..." | tee -a
$LOG
-
- # check etcd conf
- hostCount=${#ETCD_HOSTS[@]}
- if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2
-echo "Number of nodes = [$hostCount], must be configured to be greater
than or equal to 3 servers! " | tee -a $LOG
-exit_install
- fi
- for ip in ${ETCD_HOSTS[@]}
- do
-if ! valid_ip $ip; then
- echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG
- exit_install
-fi
- done
- echo "Check if the configuration file configuration is correct [ Done ]" |
tee -a $LOG
-}
-
-## @description index by EtcdHosts list
-## @audience public
-## @stabilitystable
-function indexByEtcdHosts() {
- index=0
- while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do
-if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then
- echo $index
- return
-fi
-let "index++"
- done
- echo ""
-}
-
-## @description get local IP
-## @audience public
-## @stabilitystable
-function getLocalIP()
-{
- local _ip _myip _line _nl=$'\n'
- while IFS=$': \t' read -a _line ;do
- [ -z "${_line%inet}" ] &&
- _ip=${_line[${#_line[1]}>4?1:2]} &&
- [ "${_ip#127.0.0.1}" ] && _myip=$_ip
-done< <(LANG=C /sbin/ifconfig)
- printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip
-}
-
-## @description get ip list
-## @audience public
-## @stabilitystable
-function get_ip_list()
-{
- array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[
\t]*//g' | cut -d ' ' -f2)
-
- for ip in ${array[@]}
- do
-LOCAL_HOST_IP_LIST+=(${ip})
- done
-}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-9010. Fix the incorrect trailing slash deletion in constructor method of CGroupsHandlerImpl. (Zhankun Tang via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 0081b02e3 -> bad12031f
YARN-9010. Fix the incorrect trailing slash deletion in constructor method of
CGroupsHandlerImpl. (Zhankun Tang via wangda)
Change-Id: Iaecc66d57781cc10f19ead4647e47fc9556676da
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bad12031
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bad12031
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bad12031
Branch: refs/heads/trunk
Commit: bad12031f603347a701249a1e3ef5d879a5f1c8f
Parents: 0081b02
Author: Wangda Tan
Authored: Thu Nov 29 14:56:07 2018 -0800
Committer: Wangda Tan
Committed: Thu Nov 29 14:56:07 2018 -0800
--
.../linux/resources/CGroupsHandlerImpl.java | 3 +-
.../linux/resources/TestCGroupsHandlerImpl.java | 38
2 files changed, 40 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bad12031/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index 050d0a8..1b2c780 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -87,9 +87,10 @@ class CGroupsHandlerImpl implements CGroupsHandler {
CGroupsHandlerImpl(Configuration conf, PrivilegedOperationExecutor
privilegedOperationExecutor, String mtab)
throws ResourceHandlerException {
+// Remove leading and trialing slash(es)
this.cGroupPrefix = conf.get(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, "/hadoop-yarn")
-.replaceAll("^/", "").replaceAll("$/", "");
+.replaceAll("^/+", "").replaceAll("/+$", "");
this.enableCGroupMount = conf.getBoolean(YarnConfiguration.
NM_LINUX_CONTAINER_CGROUPS_MOUNT, false);
this.cGroupMountPath = conf.get(YarnConfiguration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bad12031/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
index ea6fb52..70badaf 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
@@ -598,4 +598,42 @@ public class TestCGroupsHandlerImpl {
FileUtils.deleteQuietly(cpu);
}
}
+
+ // Remove leading and trailing slashes
+ @Test
+ public void testCgroupsHierarchySetting() throws ResourceHandlerException {
+YarnConfiguration conf = new YarnConfiguration();
+conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH, tmpPath);
+conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY,
+"/hadoop-yarn");
+CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(conf, null);
+String expectedRelativePath = "hadoop-yarn/c1";
+Assert.assertEquals(expectedRelativePath,
+cGroupsHandler.getRelativePathForCGroup("c1"));
+
+conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGR
hadoop git commit: YARN-9030. Log aggregation changes to handle filesystems which do not support setting permissions. (Suma Shivaprasad via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.2 7a78bdf7b -> ee9deb6e9
YARN-9030. Log aggregation changes to handle filesystems which do not support
setting permissions. (Suma Shivaprasad via wangda)
Change-Id: I80f1e8196b8624e24d74494719fdedfd7061dced
(cherry picked from commit 9de8e8d0496a2628b63cc841b1fdee80e2912f7a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee9deb6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee9deb6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee9deb6e
Branch: refs/heads/branch-3.2
Commit: ee9deb6e9ccebcf6ef2660e32b0d26f458b3ee5d
Parents: 7a78bdf
Author: Wangda Tan
Authored: Wed Nov 21 17:28:37 2018 -0800
Committer: Wangda Tan
Committed: Wed Nov 28 15:06:35 2018 -0800
--
.../LogAggregationFileController.java | 74 +++-
1 file changed, 56 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee9deb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index fe65288..e37308d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -109,6 +109,8 @@ public abstract class LogAggregationFileController {
protected int retentionSize;
protected String fileControllerName;
+ protected boolean fsSupportsChmod = true;
+
public LogAggregationFileController() {}
/**
@@ -250,7 +252,6 @@ public abstract class LogAggregationFileController {
* Verify and create the remote log directory.
*/
public void verifyAndCreateRemoteLogDir() {
-boolean logPermError = true;
// Checking the existence of the TLD
FileSystem remoteFS = null;
try {
@@ -264,14 +265,12 @@ public abstract class LogAggregationFileController {
try {
FsPermission perms =
remoteFS.getFileStatus(remoteRootLogDir).getPermission();
- if (!perms.equals(TLDIR_PERMISSIONS) && logPermError) {
+ if (!perms.equals(TLDIR_PERMISSIONS)) {
LOG.warn("Remote Root Log Dir [" + remoteRootLogDir
+ "] already exist, but with incorrect permissions. "
+ "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms
+ "]." + " The cluster may have problems with multiple users.");
-logPermError = false;
- } else {
-logPermError = true;
+
}
} catch (FileNotFoundException e) {
remoteExists = false;
@@ -280,15 +279,26 @@ public abstract class LogAggregationFileController {
"Failed to check permissions for dir ["
+ remoteRootLogDir + "]", e);
}
+
+Path qualified =
+remoteRootLogDir.makeQualified(remoteFS.getUri(),
+remoteFS.getWorkingDirectory());
if (!remoteExists) {
LOG.warn("Remote Root Log Dir [" + remoteRootLogDir
+ "] does not exist. Attempting to create it.");
try {
-Path qualified =
-remoteRootLogDir.makeQualified(remoteFS.getUri(),
-remoteFS.getWorkingDirectory());
remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS));
-remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS));
+
+// Not possible to query FileSystem API to check if it supports
+// chmod, chown etc. Hence resorting to catching exceptions here.
+// Remove when FS APi is ready
+try {
+ remoteFS.setPermission(qualified, new
FsPermission(TLDIR_PERMISSIONS));
+} catch ( UnsupportedOperationException use) {
+ LOG.info("Unable to set permissions for configured filesystem since"
+ + " it does not support this", remoteFS.getScheme());
+ fsSupportsChmod = false;
+}
UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
String primaryGroupName = null;
@@ -301,13 +311,31 @@ public abstract class LogAggregationFileController {
}
// set owner on th
[3/6] hadoop git commit: YARN-9061. Improve the GPU/FPGA module log message of container-executor. (Zhankun Tang via wangda)
YARN-9061. Improve the GPU/FPGA module log message of container-executor.
(Zhankun Tang via wangda)
Change-Id: Iece9b47438357077a53984a820d4d6423f480518
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ed87567
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ed87567
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ed87567
Branch: refs/heads/trunk
Commit: 9ed87567ad0f1c26a263ce6d8fba56d066260c5c
Parents: 579ef4b
Author: Wangda Tan
Authored: Wed Nov 28 14:31:31 2018 -0800
Committer: Wangda Tan
Committed: Wed Nov 28 14:31:31 2018 -0800
--
.../native/container-executor/impl/modules/fpga/fpga-module.c | 5 +++--
.../native/container-executor/impl/modules/gpu/gpu-module.c | 5 +++--
2 files changed, 6 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed87567/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
index c1a2f83..e947d7c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
@@ -141,7 +141,7 @@ void reload_fpga_configuration() {
/*
* Format of FPGA request commandline:
*
- * c-e fpga --excluded_fpgas 0,1,3 --container_id container_x_y
+ * c-e --module-fpga --excluded_fpgas 0,1,3 --container_id container_x_y
*/
int handle_fpga_request(update_cgroups_parameters_function func,
const char* module_name, int module_argc, char** module_argv) {
@@ -213,7 +213,8 @@ int handle_fpga_request(update_cgroups_parameters_function
func,
if (!minor_devices) {
// Minor devices is null, skip following call.
- fprintf(ERRORFILE, "is not specified, skip cgroups call.\n");
+ fprintf(ERRORFILE,
+ "--excluded-fpgas is not specified, skip cgroups call.\n");
goto cleanup;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed87567/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
index 1a1b164..7522338 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
@@ -141,7 +141,7 @@ void reload_gpu_configuration() {
/*
* Format of GPU request commandline:
*
- * c-e gpu --excluded_gpus 0,1,3 --container_id container_x_y
+ * c-e --module-gpu --excluded_gpus 0,1,3 --container_id container_x_y
*/
int handle_gpu_request(update_cgroups_parameters_func func,
const char* module_name, int module_argc, char** module_argv) {
@@ -213,7 +213,8 @@ int handle_gpu_request(update_cgroups_parameters_func func,
if (!minor_devices) {
// Minor devices is null, skip following call.
- fprintf(ERRORFILE, "is not specified, skip cgroups call.\n");
+ fprintf(ERRORFILE,
+ "--excluded_gpus is not specified, skip cgroups call.\n");
goto cleanup;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/6] hadoop git commit: YARN-9030. Log aggregation changes to handle filesystems which do not support setting permissions. (Suma Shivaprasad via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 4d8de7ab6 -> fe7dab8ef
YARN-9030. Log aggregation changes to handle filesystems which do not support
setting permissions. (Suma Shivaprasad via wangda)
Change-Id: I80f1e8196b8624e24d74494719fdedfd7061dced
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9de8e8d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9de8e8d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9de8e8d0
Branch: refs/heads/trunk
Commit: 9de8e8d0496a2628b63cc841b1fdee80e2912f7a
Parents: 4d8de7a
Author: Wangda Tan
Authored: Wed Nov 21 17:28:37 2018 -0800
Committer: Wangda Tan
Committed: Wed Nov 28 13:36:21 2018 -0800
--
.../LogAggregationFileController.java | 74 +++-
1 file changed, 56 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9de8e8d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index fe65288..e37308d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -109,6 +109,8 @@ public abstract class LogAggregationFileController {
protected int retentionSize;
protected String fileControllerName;
+ protected boolean fsSupportsChmod = true;
+
public LogAggregationFileController() {}
/**
@@ -250,7 +252,6 @@ public abstract class LogAggregationFileController {
* Verify and create the remote log directory.
*/
public void verifyAndCreateRemoteLogDir() {
-boolean logPermError = true;
// Checking the existence of the TLD
FileSystem remoteFS = null;
try {
@@ -264,14 +265,12 @@ public abstract class LogAggregationFileController {
try {
FsPermission perms =
remoteFS.getFileStatus(remoteRootLogDir).getPermission();
- if (!perms.equals(TLDIR_PERMISSIONS) && logPermError) {
+ if (!perms.equals(TLDIR_PERMISSIONS)) {
LOG.warn("Remote Root Log Dir [" + remoteRootLogDir
+ "] already exist, but with incorrect permissions. "
+ "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms
+ "]." + " The cluster may have problems with multiple users.");
-logPermError = false;
- } else {
-logPermError = true;
+
}
} catch (FileNotFoundException e) {
remoteExists = false;
@@ -280,15 +279,26 @@ public abstract class LogAggregationFileController {
"Failed to check permissions for dir ["
+ remoteRootLogDir + "]", e);
}
+
+Path qualified =
+remoteRootLogDir.makeQualified(remoteFS.getUri(),
+remoteFS.getWorkingDirectory());
if (!remoteExists) {
LOG.warn("Remote Root Log Dir [" + remoteRootLogDir
+ "] does not exist. Attempting to create it.");
try {
-Path qualified =
-remoteRootLogDir.makeQualified(remoteFS.getUri(),
-remoteFS.getWorkingDirectory());
remoteFS.mkdirs(qualified, new FsPermission(TLDIR_PERMISSIONS));
-remoteFS.setPermission(qualified, new FsPermission(TLDIR_PERMISSIONS));
+
+// Not possible to query FileSystem API to check if it supports
+// chmod, chown etc. Hence resorting to catching exceptions here.
+// Remove when FS APi is ready
+try {
+ remoteFS.setPermission(qualified, new
FsPermission(TLDIR_PERMISSIONS));
+} catch ( UnsupportedOperationException use) {
+ LOG.info("Unable to set permissions for configured filesystem since"
+ + " it does not support this", remoteFS.getScheme());
+ fsSupportsChmod = false;
+}
UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
String primaryGroupName = null;
@@ -301,13 +311,31 @@ public abstract class LogAggregationFileController {
}
// set owner on the remote directory only if the primary group exists
if (primaryGroupNam
[6/6] hadoop git commit: YARN-8989. [YARN-8851] Move DockerCommandPlugin volume related APIs' invocation from DockerLinuxContainerRuntime#prepareContainer to #launchContainer. (Zhankun Tang via wangda
YARN-8989. [YARN-8851] Move DockerCommandPlugin volume related APIs' invocation
from DockerLinuxContainerRuntime#prepareContainer to #launchContainer. (Zhankun
Tang via wangda)
Change-Id: Ia6d532c687168448416dfdf46f0ac34bff20e6ca
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe7dab8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe7dab8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe7dab8e
Branch: refs/heads/trunk
Commit: fe7dab8ef55f08cf18c2d62c782c1ab8930a5a15
Parents: 8976439
Author: Wangda Tan
Authored: Wed Nov 28 14:55:16 2018 -0800
Committer: Wangda Tan
Committed: Wed Nov 28 15:03:06 2018 -0800
--
.../runtime/DockerLinuxContainerRuntime.java| 44
.../runtime/TestDockerContainerRuntime.java | 15 ---
2 files changed, 24 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe7dab8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 15ff0ff..225bc19 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -456,32 +456,6 @@ public class DockerLinuxContainerRuntime implements
LinuxContainerRuntime {
@Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
-Container container = ctx.getContainer();
-
-// Create volumes when needed.
-if (nmContext != null
-&& nmContext.getResourcePluginManager().getNameToPlugins() != null) {
- for (ResourcePlugin plugin : nmContext.getResourcePluginManager()
- .getNameToPlugins().values()) {
-DockerCommandPlugin dockerCommandPlugin =
-plugin.getDockerCommandPluginInstance();
-if (dockerCommandPlugin != null) {
- DockerVolumeCommand dockerVolumeCommand =
- dockerCommandPlugin.getCreateDockerVolumeCommand(
- ctx.getContainer());
- if (dockerVolumeCommand != null) {
-runDockerVolumeCommand(dockerVolumeCommand, container);
-
-// After volume created, run inspect to make sure volume properly
-// created.
-if (dockerVolumeCommand.getSubCommand().equals(
-DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND)) {
- checkDockerVolumeCreated(dockerVolumeCommand, container);
-}
- }
-}
- }
-}
}
private void checkDockerVolumeCreated(
@@ -1034,14 +1008,30 @@ public class DockerLinuxContainerRuntime implements
LinuxContainerRuntime {
}
}
-// use plugins to update docker run command.
+// use plugins to create volume and update docker run command.
if (nmContext != null
&& nmContext.getResourcePluginManager().getNameToPlugins() != null) {
for (ResourcePlugin plugin : nmContext.getResourcePluginManager()
.getNameToPlugins().values()) {
DockerCommandPlugin dockerCommandPlugin =
plugin.getDockerCommandPluginInstance();
+
if (dockerCommandPlugin != null) {
+ // Create volumes when needed.
+ DockerVolumeCommand dockerVolumeCommand =
+ dockerCommandPlugin.getCreateDockerVolumeCommand(
+ ctx.getContainer());
+ if (dockerVolumeCommand != null) {
+runDockerVolumeCommand(dockerVolumeCommand, container);
+
+// After volume created, run inspect to make sure volume properly
+// created.
+if (dockerVolumeCommand.getSubCommand().equals(
+DockerVolumeCommand.VOLUME_CREATE_SUB_COMMAND)) {
+ checkDockerVolumeCreated(dockerVolumeCommand, container);
+}
+ }
+ // Update cmd
dockerCommandPlugin.updateDockerRunCommand(runCommand, container);
[2/6] hadoop git commit: YARN-8882. [YARN-8851] Add a shared device mapping manager (scheduler) for device plugins. (Zhankun Tang via wangda)
YARN-8882. [YARN-8851] Add a shared device mapping manager (scheduler) for
device plugins. (Zhankun Tang via wangda)
Change-Id: I9435136642c3d556971a357bf687f69df90bb45e
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/579ef4be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/579ef4be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/579ef4be
Branch: refs/heads/trunk
Commit: 579ef4be063745c5211127eca83a393ceddc8b79
Parents: 9de8e8d
Author: Wangda Tan
Authored: Wed Nov 28 14:09:52 2018 -0800
Committer: Wangda Tan
Committed: Wed Nov 28 14:09:52 2018 -0800
--
.../resourceplugin/ResourcePluginManager.java | 14 +-
.../deviceframework/DeviceMappingManager.java | 324
.../deviceframework/DevicePluginAdapter.java| 20 +-
.../DeviceResourceHandlerImpl.java | 145 +++
.../TestDeviceMappingManager.java | 366 +
.../TestDevicePluginAdapter.java| 388 ++-
6 files changed, 1245 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/579ef4be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
index 9741b12..6dfe817 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin;
import
org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRegisterRequest;
+import
org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.DeviceMappingManager;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework.DevicePluginAdapter;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaResourcePlugin;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin;
@@ -52,12 +53,13 @@ import static
org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI;
public class ResourcePluginManager {
private static final Logger LOG =
LoggerFactory.getLogger(ResourcePluginManager.class);
- private static final Set SUPPORTED_RESOURCE_PLUGINS =
ImmutableSet.of(
- GPU_URI, FPGA_URI);
+ private static final Set SUPPORTED_RESOURCE_PLUGINS =
+ ImmutableSet.of(GPU_URI, FPGA_URI);
private Map configuredPlugins =
Collections.emptyMap();
+ private DeviceMappingManager deviceMappingManager = null;
public synchronized void initialize(Context context)
throws YarnException, ClassNotFoundException {
@@ -123,7 +125,7 @@ public class ResourcePluginManager {
throws YarnRuntimeException, ClassNotFoundException {
LOG.info("The pluggable device framework enabled," +
"trying to load the vendor plugins");
-
+deviceMappingManager = new DeviceMappingManager(context);
String[] pluginClassNames = configuration.getStrings(
YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES);
if (null == pluginClassNames) {
@@ -174,7 +176,7 @@ public class ResourcePluginManager {
resourceName,
pluginClassName);
DevicePluginAdapter pluginAdapter = new DevicePluginAdapter(
- resourceName, dpInstance);
+ resourceName, dpInstance, deviceMappingManager);
LOG.info("Adapter of {} created. Initializing..", pluginClassName);
try {
pluginAdapter.initialize(context);
@@ -235,6 +237,10 @@ public class ResourcePluginManager {
return true;
}
+ public DeviceMappingManager getDeviceMappingManager() {
+return deviceMappingManager;
+ }
+
public synchronized void
[5/6] hadoop git commit: YARN-8975. [Submarine] Use predefined Charset object StandardCharsets.UTF_8 instead of String UTF-8. (Zhankun Tang via wangda)
YARN-8975. [Submarine] Use predefined Charset object StandardCharsets.UTF_8
instead of String UTF-8. (Zhankun Tang via wangda)
Change-Id: If6c7904aa17895e543cfca245264249eb7328bdc
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89764392
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89764392
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89764392
Branch: refs/heads/trunk
Commit: 897643928c534062d45d00a36b95fd99b4f6
Parents: 8ebeda9
Author: Wangda Tan
Authored: Wed Nov 28 14:39:06 2018 -0800
Committer: Wangda Tan
Committed: Wed Nov 28 14:39:06 2018 -0800
--
.../submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89764392/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
index b58ad77..2e84c96 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
@@ -49,6 +49,7 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -218,7 +219,8 @@ public class YarnServiceJobSubmitter implements
JobSubmitter {
private String generateCommandLaunchScript(RunJobParameters parameters,
TaskType taskType, Component comp) throws IOException {
File file = File.createTempFile(taskType.name() + "-launch-script", ".sh");
-Writer w = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
+Writer w = new OutputStreamWriter(new FileOutputStream(file),
+StandardCharsets.UTF_8);
PrintWriter pw = new PrintWriter(w);
try {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-8960. [Submarine] Can't get submarine service status using the command of "yarn app -status" under security environment. (Zac Zhou via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.2 0b2cfc8ab -> 34387599c
YARN-8960. [Submarine] Can't get submarine service status using the command of
"yarn app -status" under security environment. (Zac Zhou via wangda)
Change-Id: I21b1addc9c32817650ea744a8f2e6b5602f2f4d4
(cherry picked from commit 8b2381441558cd49b4c940b0760c8accbb2a5567)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34387599
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34387599
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34387599
Branch: refs/heads/branch-3.2
Commit: 34387599c41b5026a7ab179b31fbc09cfcbd5071
Parents: 0b2cfc8
Author: Wangda Tan
Authored: Mon Nov 19 08:55:30 2018 -0800
Committer: Wangda Tan
Committed: Mon Nov 19 09:01:21 2018 -0800
--
.../hadoop/yarn/submarine/client/cli/Cli.java | 3 -
.../yarn/submarine/client/cli/CliConstants.java | 3 +
.../yarn/submarine/client/cli/CliUtils.java | 56 ++
.../yarn/submarine/client/cli/RunJobCli.java| 12 +++-
.../client/cli/param/RunJobParameters.java | 46 ++-
.../yarn/submarine/common/ClientContext.java| 15 +++--
.../fs/DefaultRemoteDirectoryManager.java | 3 +-
.../yarnservice/YarnServiceJobSubmitter.java| 61 +---
.../client/cli/TestRunJobCliParsing.java| 9 ++-
9 files changed, 188 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34387599/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java
index b4c5e4c..69189f4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/Cli.java
@@ -16,7 +16,6 @@ package org.apache.hadoop.yarn.submarine.client.cli;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.submarine.common.ClientContext;
-import
org.apache.hadoop.yarn.submarine.common.fs.DefaultRemoteDirectoryManager;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.submarine.runtimes.RuntimeFactory;
import org.slf4j.Logger;
@@ -43,8 +42,6 @@ public class Cli {
Configuration conf = new YarnConfiguration();
ClientContext clientContext = new ClientContext();
clientContext.setConfiguration(conf);
-clientContext.setRemoteDirectoryManager(
-new DefaultRemoteDirectoryManager(clientContext));
RuntimeFactory runtimeFactory = RuntimeFactory.getRuntimeFactory(
clientContext);
clientContext.setRuntimeFactory(runtimeFactory);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34387599/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
index 454ff1c..2d7a472 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
@@ -52,4 +52,7 @@ public class CliConstants {
public static final String QUICKLINK = "quicklink";
public static final String TENSORBOARD_DOCKER_IMAGE =
"tensorboard_docker_image";
+ public static final String KEYTAB = "keytab";
+ public static final String PRINCIPAL = "principal";
+ public static final String DISTRIBUTE_KEYTAB = "distribute_keytab";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34387599/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main
hadoop git commit: YARN-8299. Yarn Service Upgrade: Add GET APIs that returns instances matching query params. (Chandni Singh via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 095635d98 -> 077a7ab37
YARN-8299. Yarn Service Upgrade: Add GET APIs that returns instances matching
query params. (Chandni Singh via wangda)
Change-Id: I7f2aef89c6053d5dc5955f79fdfcd4cd74f74737
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/077a7ab3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/077a7ab3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/077a7ab3
Branch: refs/heads/branch-3.1
Commit: 077a7ab37bd59ae63743dc94020cdaf5c62af396
Parents: 095635d
Author: Wangda Tan
Authored: Fri Nov 16 10:32:10 2018 -0800
Committer: Wangda Tan
Committed: Fri Nov 16 10:32:10 2018 -0800
--
.../yarn/service/client/ApiServiceClient.java | 73 ++---
.../hadoop/yarn/service/webapp/ApiServer.java | 67 ++--
.../hadoop/yarn/service/ClientAMProtocol.java | 5 +
.../hadoop/yarn/service/ClientAMService.java| 14 ++
.../yarn/service/client/ServiceClient.java | 47 ++
.../component/instance/ComponentInstance.java | 41 -
.../yarn/service/conf/RestApiConstants.java | 5 +-
.../pb/client/ClientAMProtocolPBClientImpl.java | 13 ++
.../service/ClientAMProtocolPBServiceImpl.java | 13 ++
.../hadoop/yarn/service/utils/FilterUtils.java | 81 ++
.../yarn/service/utils/ServiceApiUtil.java | 9 ++
.../src/main/proto/ClientAMProtocol.proto | 12 ++
.../yarn/service/MockRunningServiceContext.java | 154 +++
.../yarn/service/client/TestServiceCLI.java | 25 ++-
.../yarn/service/client/TestServiceClient.java | 54 ++-
.../yarn/service/component/TestComponent.java | 133 +---
.../instance/TestComponentInstance.java | 46 +++---
.../yarn/service/utils/TestFilterUtils.java | 102
.../hadoop/yarn/client/cli/ApplicationCLI.java | 68 +++-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 6 +-
.../hadoop/yarn/client/api/AppAdminClient.java | 6 +
21 files changed, 772 insertions(+), 202 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/077a7ab3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index 074b45d..e0b3d86 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -29,6 +29,7 @@ import java.util.Map;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriBuilder;
import com.google.common.base.Preconditions;
import org.apache.commons.codec.binary.Base64;
@@ -55,10 +56,8 @@ import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
import org.apache.hadoop.yarn.service.conf.RestApiConstants;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.util.RMHAUtils;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
import org.eclipse.jetty.util.UrlEncoded;
import org.ietf.jgss.GSSContext;
import org.ietf.jgss.GSSException;
@@ -216,11 +215,7 @@ public class ApiServiceClient extends AppAdminClient {
api.append("/");
api.append(appName);
}
-Configuration conf = getConfig();
-if
(conf.get("hadoop.http.authentication.type").equalsIgnoreCase("simple")) {
- api.append("?user.name=" + UrlEncoded
- .encodeString(System.getProperty("user.name")));
-}
+appendUserNameIfRequired(api);
return api.toString();
}
@@ -231,15 +226,27 @@ public class ApiServiceClient extends AppAdminClient {
api.append(url);
api.append("/app/v1/services/").append(appName).append("/")
.append(RestApiConstants.COMP_INSTANCES);
-Configuration conf = getConfig();
-if (conf.get("hadoop.http.a
hadoop git commit: YARN-9001. [Submarine] Use AppAdminClient instead of ServiceClient to sumbit jobs. (Zac Zhou via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.2 f5e1bad0f -> bee5bf867
YARN-9001. [Submarine] Use AppAdminClient instead of ServiceClient to sumbit
jobs. (Zac Zhou via wangda)
Change-Id: I7e8d1c27ebd37e0907ca570c4f3d56fe7a859635
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bee5bf86
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bee5bf86
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bee5bf86
Branch: refs/heads/branch-3.2
Commit: bee5bf867ba6708f67bb368e2e32c5df538a2e70
Parents: f5e1bad
Author: Wangda Tan
Authored: Fri Nov 16 10:26:50 2018 -0800
Committer: Wangda Tan
Committed: Fri Nov 16 10:26:50 2018 -0800
--
.../yarn/service/client/ServiceClient.java | 1 +
.../submarine/runtimes/common/JobMonitor.java | 6 +++
.../yarnservice/YarnServiceJobMonitor.java | 25 +---
.../yarnservice/YarnServiceJobSubmitter.java| 42 ++--
.../runtimes/yarnservice/YarnServiceUtils.java | 15 +++
.../yarnservice/TestYarnServiceRunJobCli.java | 12 +++---
.../submarine/common/MockClientContext.java | 1 -
7 files changed, 79 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bee5bf86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 8724a87..e70e96b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -1360,6 +1360,7 @@ public class ServiceClient extends AppAdminClient
implements SliderExitCodes,
LOG.info("Service {} does not have an application ID", serviceName);
return appSpec;
}
+appSpec.setId(currentAppId.toString());
ApplicationReport appReport =
yarnClient.getApplicationReport(currentAppId);
appSpec.setState(convertState(appReport.getYarnApplicationState()));
ApplicationTimeout lifetime =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bee5bf86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java
index c81393b..35e21fc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java
@@ -48,6 +48,11 @@ public abstract class JobMonitor {
throws IOException, YarnException;
/**
+ * Cleanup AppAdminClient, etc.
+ */
+ public void cleanup() throws IOException {}
+
+ /**
* Continue wait and print status if job goes to ready or final state.
* @param jobName
* @throws IOException
@@ -80,5 +85,6 @@ public abstract class JobMonitor {
throw new IOException(e);
}
}
+cleanup();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bee5bf86/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobMonitor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobMonitor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobMonitor.java
hadoop git commit: YARN-9001. [Submarine] Use AppAdminClient instead of ServiceClient to sumbit jobs. (Zac Zhou via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 9da6054ca -> fcbd205cc
YARN-9001. [Submarine] Use AppAdminClient instead of ServiceClient to sumbit
jobs. (Zac Zhou via wangda)
Change-Id: Ic3d6c1e439df9cdf74448b345b925343224efe51
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcbd205c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcbd205c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcbd205c
Branch: refs/heads/trunk
Commit: fcbd205cc35e7411ac33860c78b9e1e70697bb4a
Parents: 9da6054
Author: Wangda Tan
Authored: Tue Nov 13 13:13:27 2018 -0800
Committer: Wangda Tan
Committed: Tue Nov 13 13:13:27 2018 -0800
--
.../yarn/service/client/ServiceClient.java | 1 +
.../submarine/runtimes/common/JobMonitor.java | 6 +++
.../yarnservice/YarnServiceJobMonitor.java | 27 +
.../yarnservice/YarnServiceJobSubmitter.java| 42 ++--
.../runtimes/yarnservice/YarnServiceUtils.java | 15 +++
.../yarnservice/TestYarnServiceRunJobCli.java | 12 +++---
.../submarine/common/MockClientContext.java | 1 -
7 files changed, 80 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcbd205c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 0bc5a2c..713d890 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -1552,6 +1552,7 @@ public class ServiceClient extends AppAdminClient
implements SliderExitCodes,
LOG.info("Service {} does not have an application ID", serviceName);
return appSpec;
}
+appSpec.setId(currentAppId.toString());
ApplicationReport appReport =
yarnClient.getApplicationReport(currentAppId);
appSpec.setState(convertState(appReport.getYarnApplicationState()));
ApplicationTimeout lifetime =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcbd205c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java
index c81393b..35e21fc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/JobMonitor.java
@@ -48,6 +48,11 @@ public abstract class JobMonitor {
throws IOException, YarnException;
/**
+ * Cleanup AppAdminClient, etc.
+ */
+ public void cleanup() throws IOException {}
+
+ /**
* Continue wait and print status if job goes to ready or final state.
* @param jobName
* @throws IOException
@@ -80,5 +85,6 @@ public abstract class JobMonitor {
throw new IOException(e);
}
}
+cleanup();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcbd205c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobMonitor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobMonitor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobMonitor.java
index fab
hadoop git commit: MAPREDUCE-7158. Inefficient Flush Logic in JobHistory EventWriter. (Zichen Sun via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 c97f8b6d9 -> abd9d93a5
MAPREDUCE-7158. Inefficient Flush Logic in JobHistory EventWriter. (Zichen Sun
via wangda)
Change-Id: I99ace87980da03bb35a8012cea7218d602a8817a
(cherry picked from commit 762a56cc64bc07d57f94e253920534b8e049f238)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/abd9d93a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/abd9d93a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/abd9d93a
Branch: refs/heads/branch-3.1
Commit: abd9d93a5544a3172c9d9a48385c2ce7fc13fa34
Parents: c97f8b6
Author: Wangda Tan
Authored: Tue Nov 13 11:25:41 2018 -0800
Committer: Wangda Tan
Committed: Tue Nov 13 11:36:04 2018 -0800
--
.../java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java| 1 -
1 file changed, 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/abd9d93a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
index 46e4f1a..b07c676 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
@@ -93,7 +93,6 @@ public class EventWriter {
wrapper.setType(event.getEventType());
wrapper.setEvent(event.getDatum());
writer.write(wrapper, encoder);
-encoder.flush();
if (this.jsonOutput) {
out.writeBytes("\n");
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: MAPREDUCE-7158. Inefficient Flush Logic in JobHistory EventWriter. (Zichen Sun via wangda)
MAPREDUCE-7158. Inefficient Flush Logic in JobHistory EventWriter. (Zichen Sun
via wangda)
Change-Id: I99ace87980da03bb35a8012cea7218d602a8817a
(cherry picked from commit 762a56cc64bc07d57f94e253920534b8e049f238)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f214af74
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f214af74
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f214af74
Branch: refs/heads/branch-3.2
Commit: f214af74b29beb4aa76fde30d81cfb6e8c59f694
Parents: ea75304
Author: Wangda Tan
Authored: Tue Nov 13 11:25:41 2018 -0800
Committer: Wangda Tan
Committed: Tue Nov 13 11:35:13 2018 -0800
--
.../java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java| 1 -
1 file changed, 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f214af74/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
index 46e4f1a..b07c676 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
@@ -93,7 +93,6 @@ public class EventWriter {
wrapper.setType(event.getEventType());
wrapper.setEvent(event.getDatum());
writer.write(wrapper, encoder);
-encoder.flush();
if (this.jsonOutput) {
out.writeBytes("\n");
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/2] hadoop git commit: YARN-8918. [Submarine] Correct method usage of str.subString in CliUtils. (Zhankun Tang via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.2 86deff0a0 -> f214af74b
YARN-8918. [Submarine] Correct method usage of str.subString in CliUtils.
(Zhankun Tang via wangda)
Change-Id: Id1f11dbab3aa838dee3a0ec8b8fd5dc32f5dd946
(cherry picked from commit 076b795b2e82d7f89cc91e0a8513c7081ee8b930)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea753046
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea753046
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea753046
Branch: refs/heads/branch-3.2
Commit: ea753046b0f418dc850db618f1e2d8b4edeb3f81
Parents: 86deff0
Author: Wangda Tan
Authored: Tue Nov 13 11:24:15 2018 -0800
Committer: Wangda Tan
Committed: Tue Nov 13 11:35:06 2018 -0800
--
.../java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea753046/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
index 546c6eb..bfdfa9a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
@@ -72,7 +72,7 @@ public class CliUtils {
resourcesStr = resourcesStr.substring(1);
}
if (resourcesStr.endsWith("]")) {
- resourcesStr = resourcesStr.substring(0, resourcesStr.length());
+ resourcesStr = resourcesStr.substring(0, resourcesStr.length() - 1);
}
for (String resource : resourcesStr.trim().split(",")) {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: MAPREDUCE-7158. Inefficient Flush Logic in JobHistory EventWriter. (Zichen Sun via wangda)
MAPREDUCE-7158. Inefficient Flush Logic in JobHistory EventWriter. (Zichen Sun
via wangda)
Change-Id: I99ace87980da03bb35a8012cea7218d602a8817a
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/762a56cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/762a56cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/762a56cc
Branch: refs/heads/trunk
Commit: 762a56cc64bc07d57f94e253920534b8e049f238
Parents: 076b795
Author: Wangda Tan
Authored: Tue Nov 13 11:25:41 2018 -0800
Committer: Wangda Tan
Committed: Tue Nov 13 11:25:41 2018 -0800
--
.../java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java| 1 -
1 file changed, 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/762a56cc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
index 46e4f1a..b07c676 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
@@ -93,7 +93,6 @@ public class EventWriter {
wrapper.setType(event.getEventType());
wrapper.setEvent(event.getDatum());
writer.write(wrapper, encoder);
-encoder.flush();
if (this.jsonOutput) {
out.writeBytes("\n");
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/2] hadoop git commit: YARN-8918. [Submarine] Correct method usage of str.subString in CliUtils. (Zhankun Tang via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 671fd6524 -> 762a56cc6
YARN-8918. [Submarine] Correct method usage of str.subString in CliUtils.
(Zhankun Tang via wangda)
Change-Id: Id1f11dbab3aa838dee3a0ec8b8fd5dc32f5dd946
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/076b795b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/076b795b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/076b795b
Branch: refs/heads/trunk
Commit: 076b795b2e82d7f89cc91e0a8513c7081ee8b930
Parents: 671fd65
Author: Wangda Tan
Authored: Tue Nov 13 11:24:15 2018 -0800
Committer: Wangda Tan
Committed: Tue Nov 13 11:24:15 2018 -0800
--
.../java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/076b795b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
index 05e830f..f85c82a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
@@ -72,7 +72,7 @@ public class CliUtils {
resourcesStr = resourcesStr.substring(1);
}
if (resourcesStr.endsWith("]")) {
- resourcesStr = resourcesStr.substring(0, resourcesStr.length());
+ resourcesStr = resourcesStr.substring(0, resourcesStr.length() - 1);
}
for (String resource : resourcesStr.trim().split(",")) {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-8908. Fix errors in yarn-default.xml related to GPU/FPGA. (Zhankun Tang via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 bad5d81ac -> c11d2341a
YARN-8908. Fix errors in yarn-default.xml related to GPU/FPGA. (Zhankun Tang
via wangda)
Change-Id: I46f511a9b51b65b069e15d5b827530110c41edab
(cherry picked from commit 071b9e19aa2c17bb9fa046c6ddabd6b8f0cc6d14)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c11d2341
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c11d2341
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c11d2341
Branch: refs/heads/branch-3.1
Commit: c11d2341afe1da8a98c1d0f048147dc664fc8ab3
Parents: bad5d81
Author: Wangda Tan
Authored: Fri Oct 19 09:44:00 2018 -0700
Committer: Wangda Tan
Committed: Fri Oct 19 09:55:27 2018 -0700
--
.../hadoop-yarn-common/src/main/resources/yarn-default.xml | 5 +++--
.../hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md | 2 +-
2 files changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c11d2341/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 412fd02..fd68804 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3634,7 +3634,7 @@
Enable additional discovery/isolation of resources on the NodeManager,
split by comma. By default, this is empty.
- Acceptable values: { "yarn-io/gpu", "yarn-io/fpga"}.
+ Acceptable values: { "yarn.io/gpu", "yarn.io/fpga"}.
yarn.nodemanager.resource-plugins
@@ -3715,9 +3715,10 @@
At present, since we can only configure one major number in c-e.cfg,
FPGA device is
identified by their minor device number. A common approach to get minor
device number of FPGA is using "aocl diagnose" and check uevent with
device name.
+ A sample manual value for this is like "0,1"
yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices
-0,1
+auto
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c11d2341/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
index d98bb15..bc0589a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
@@ -53,7 +53,7 @@ Use following property to configure
`DominantResourceCalculator` (In `capacity-s
```
yarn.nodemanager.resource-plugins
-yarn-io/fpga
+yarn.io/fpga
```
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: YARN-8916. Define a constant docker string in ContainerRuntimeConstants.java for better maintainability. (Zhankun Tang via wangda)
YARN-8916. Define a constant docker string in ContainerRuntimeConstants.java
for better maintainability. (Zhankun Tang via wangda)
Change-Id: I1349e740037f81afdbe30edbe741f20e88fd0a90
(cherry picked from commit 5e02b4915b876eab943b30cf728cfc26ba832188)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ed9e185
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ed9e185
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ed9e185
Branch: refs/heads/branch-3.2
Commit: 9ed9e185d75ba303e04fd4a675c59dbc28670faf
Parents: bd08554
Author: Wangda Tan
Authored: Fri Oct 19 09:44:45 2018 -0700
Committer: Wangda Tan
Committed: Fri Oct 19 09:50:18 2018 -0700
--
.../linux/runtime/DockerLinuxContainerRuntime.java | 3 ++-
.../runtime/ContainerRuntimeConstants.java | 4
.../containermanager/container/TestContainer.java | 3 ++-
.../linux/resources/gpu/TestGpuResourceHandler.java | 3 ++-
.../runtime/TestDelegatingLinuxContainerRuntime.java| 5 +++--
.../linux/runtime/TestDockerContainerRuntime.java | 12
6 files changed, 21 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9e185/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index c86ed53..91a348d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -266,7 +266,8 @@ public class DockerLinuxContainerRuntime implements
LinuxContainerRuntime {
if (type == null) {
type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
}
-return type != null && type.equals("docker");
+return type != null && type.equals(
+ContainerRuntimeConstants.CONTAINER_RUNTIME_DOCKER);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9e185/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java
index 4473856..a8b295d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java
@@ -30,4 +30,8 @@ public class ContainerRuntimeConstants {
@Private
public static final String ENV_CONTAINER_TYPE =
"YARN_CONTAINER_RUNTIME_TYPE";
+
+ @Private
+ public static final String CONTAINER_RUNTIME_DOCKER =
+ "docker";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9e185/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/
[1/2] hadoop git commit: YARN-8908. Fix errors in yarn-default.xml related to GPU/FPGA. (Zhankun Tang via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.2 ef073547d -> 9ed9e185d
YARN-8908. Fix errors in yarn-default.xml related to GPU/FPGA. (Zhankun Tang
via wangda)
Change-Id: I46f511a9b51b65b069e15d5b827530110c41edab
(cherry picked from commit 071b9e19aa2c17bb9fa046c6ddabd6b8f0cc6d14)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd08554a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd08554a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd08554a
Branch: refs/heads/branch-3.2
Commit: bd08554aa4220d0a5922cb08b4eaa246d09e7854
Parents: ef07354
Author: Wangda Tan
Authored: Fri Oct 19 09:44:00 2018 -0700
Committer: Wangda Tan
Committed: Fri Oct 19 09:50:12 2018 -0700
--
.../hadoop-yarn-common/src/main/resources/yarn-default.xml | 5 +++--
.../hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md | 2 +-
2 files changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd08554a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e6f7b37..ba5a96e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3764,7 +3764,7 @@
Enable additional discovery/isolation of resources on the NodeManager,
split by comma. By default, this is empty.
- Acceptable values: { "yarn-io/gpu", "yarn-io/fpga"}.
+ Acceptable values: { "yarn.io/gpu", "yarn.io/fpga"}.
yarn.nodemanager.resource-plugins
@@ -3845,9 +3845,10 @@
At present, since we can only configure one major number in c-e.cfg,
FPGA device is
identified by their minor device number. A common approach to get minor
device number of FPGA is using "aocl diagnose" and check uevent with
device name.
+ A sample manual value for this is like "0,1"
yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices
-0,1
+auto
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd08554a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
index d98bb15..bc0589a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
@@ -53,7 +53,7 @@ Use following property to configure
`DominantResourceCalculator` (In `capacity-s
```
yarn.nodemanager.resource-plugins
-yarn-io/fpga
+yarn.io/fpga
```
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: YARN-8908. Fix errors in yarn-default.xml related to GPU/FPGA. (Zhankun Tang via wangda)
YARN-8908. Fix errors in yarn-default.xml related to GPU/FPGA. (Zhankun Tang
via wangda)
Change-Id: I46f511a9b51b65b069e15d5b827530110c41edab
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/071b9e19
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/071b9e19
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/071b9e19
Branch: refs/heads/trunk
Commit: 071b9e19aa2c17bb9fa046c6ddabd6b8f0cc6d14
Parents: cd05791
Author: Wangda Tan
Authored: Fri Oct 19 09:44:00 2018 -0700
Committer: Wangda Tan
Committed: Fri Oct 19 09:49:26 2018 -0700
--
.../hadoop-yarn-common/src/main/resources/yarn-default.xml | 5 +++--
.../hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md | 2 +-
2 files changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/071b9e19/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 2ea6bfc..1360e73 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3787,7 +3787,7 @@
Enable additional discovery/isolation of resources on the NodeManager,
split by comma. By default, this is empty.
- Acceptable values: { "yarn-io/gpu", "yarn-io/fpga"}.
+ Acceptable values: { "yarn.io/gpu", "yarn.io/fpga"}.
yarn.nodemanager.resource-plugins
@@ -3868,9 +3868,10 @@
At present, since we can only configure one major number in c-e.cfg,
FPGA device is
identified by their minor device number. A common approach to get minor
device number of FPGA is using "aocl diagnose" and check uevent with
device name.
+ A sample manual value for this is like "0,1"
yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices
-0,1
+auto
http://git-wip-us.apache.org/repos/asf/hadoop/blob/071b9e19/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
index d98bb15..bc0589a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/UsingFPGA.md
@@ -53,7 +53,7 @@ Use following property to configure
`DominantResourceCalculator` (In `capacity-s
```
yarn.nodemanager.resource-plugins
-yarn-io/fpga
+yarn.io/fpga
```
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/2] hadoop git commit: YARN-8916. Define a constant docker string in ContainerRuntimeConstants.java for better maintainability. (Zhankun Tang via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk cd057918d -> 5e02b4915
YARN-8916. Define a constant docker string in ContainerRuntimeConstants.java
for better maintainability. (Zhankun Tang via wangda)
Change-Id: I1349e740037f81afdbe30edbe741f20e88fd0a90
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e02b491
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e02b491
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e02b491
Branch: refs/heads/trunk
Commit: 5e02b4915b876eab943b30cf728cfc26ba832188
Parents: 071b9e1
Author: Wangda Tan
Authored: Fri Oct 19 09:44:45 2018 -0700
Committer: Wangda Tan
Committed: Fri Oct 19 09:49:26 2018 -0700
--
.../linux/runtime/DockerLinuxContainerRuntime.java | 3 ++-
.../runtime/ContainerRuntimeConstants.java | 4
.../containermanager/container/TestContainer.java | 3 ++-
.../linux/resources/gpu/TestGpuResourceHandler.java | 3 ++-
.../runtime/TestDelegatingLinuxContainerRuntime.java| 5 +++--
.../linux/runtime/TestDockerContainerRuntime.java | 12
6 files changed, 21 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e02b491/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 21660c1..436c0ad 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -266,7 +266,8 @@ public class DockerLinuxContainerRuntime implements
LinuxContainerRuntime {
if (type == null) {
type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
}
-return type != null && type.equals("docker");
+return type != null && type.equals(
+ContainerRuntimeConstants.CONTAINER_RUNTIME_DOCKER);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e02b491/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java
index 4473856..a8b295d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntimeConstants.java
@@ -30,4 +30,8 @@ public class ContainerRuntimeConstants {
@Private
public static final String ENV_CONTAINER_TYPE =
"YARN_CONTAINER_RUNTIME_TYPE";
+
+ @Private
+ public static final String CONTAINER_RUNTIME_DOCKER =
+ "docker";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e02b491/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodema
[2/2] hadoop git commit: YARN-6098. Add documentation for Delete Queue. (Suma Shivaprasad via wangda)
YARN-6098. Add documentation for Delete Queue. (Suma Shivaprasad via wangda) Change-Id: I23bf7ca0acbe1168a7f805ceccdd62fe41bfeb35 (cherry picked from commit bfb88b10f46a265aa38ab3e1d87b6a0a99d94be8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d381f322 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d381f322 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d381f322 Branch: refs/heads/branch-3.1 Commit: d381f32219dbe27b629ba3bea3424f9648287c97 Parents: 46baafe Author: Wangda Tan Authored: Thu Oct 18 10:55:39 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 12:29:28 2018 -0700 -- .../src/site/markdown/CapacityScheduler.md| 14 +- 1 file changed, 13 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d381f322/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 9857010..75d7954 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -54,7 +54,7 @@ The `CapacityScheduler` supports the following features: * **Operability** -* Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime. +* Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime unless the queue is STOPPED and nhas no pending/running apps. * Drain applications - Administrators can *stop* queues at runtime to ensure that while existing applications run to completion, no new applications can be submitted. If a queue is in `STOPPED` state, new applications cannot be submitted to *itself* or *any of its child queues*. Existing applications continue to completion, thus the queue can be *drained* gracefully. Administrators can also *start* the stopped queues. @@ -430,6 +430,18 @@ Changing queue/scheduler properties and adding/removing queues can be done in tw $ vi $HADOOP_CONF_DIR/capacity-scheduler.xml $ $HADOOP_YARN_HOME/bin/yarn rmadmin -refreshQueues + Deleting queue via file + + Step 1: Stop the queue + + Before deleting a leaf queue, the leaf queue should not have any running/pending apps and has to BE STOPPED by changing `yarn.scheduler.capacity..state`. See the + [Queue Administration & Permissions](CapacityScheduler.html#Queue Properties) section. + Before deleting a parent queue, all its child queues should not have any running/pending apps and have to BE STOPPED. The parent queue also needs to be STOPPED + + Step 2: Delete the queue + + Remove the queue configurations from the file and run refresh as described above + ### Changing queue configuration via API Editing by API uses a backing store for the scheduler configuration. To enable this, the following parameters can be configured in yarn-site.xml. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/2] hadoop git commit: YARN-8896. Limit the maximum number of container assignments per heartbeat. (Zhankun Tang via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.1 3cb52958b -> d381f3221 YARN-8896. Limit the maximum number of container assignments per heartbeat. (Zhankun Tang via wangda) Change-Id: I6e72f8362bd7f5c2a844cb9e3c4732492314e9f1 (cherry picked from commit 780be14f07df2a3ed6273b96ae857c278fd72718) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46baafed Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46baafed Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46baafed Branch: refs/heads/branch-3.1 Commit: 46baafedf13446f5c1b973d667f437bf7158d3c2 Parents: 3cb5295 Author: Wangda Tan Authored: Thu Oct 18 10:58:21 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 12:29:19 2018 -0700 -- .../scheduler/capacity/CapacitySchedulerConfiguration.java | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/46baafed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 5c9904b..547d131 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -333,8 +333,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public static final String MAX_ASSIGN_PER_HEARTBEAT = PREFIX + "per-node-heartbeat.maximum-container-assignments"; + /** + * Avoid potential risk that greedy assign multiple may involve + * */ @Private - public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1; + public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = 100; /** Configuring absolute min/max resources in a queue. **/ @Private - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[3/3] hadoop git commit: YARN-8896. Limit the maximum number of container assignments per heartbeat. (Zhankun Tang via wangda)
YARN-8896. Limit the maximum number of container assignments per heartbeat. (Zhankun Tang via wangda) Change-Id: I6e72f8362bd7f5c2a844cb9e3c4732492314e9f1 (cherry picked from commit 780be14f07df2a3ed6273b96ae857c278fd72718) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7252f8e1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7252f8e1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7252f8e1 Branch: refs/heads/branch-3.2 Commit: 7252f8e117bacdd3b746021d4b5d6b9c944813bd Parents: 5f7ed04 Author: Wangda Tan Authored: Thu Oct 18 10:58:21 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 12:12:19 2018 -0700 -- .../scheduler/capacity/CapacitySchedulerConfiguration.java | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7252f8e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index b937ae7..08380f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -336,8 +336,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public static final String MAX_ASSIGN_PER_HEARTBEAT = PREFIX + "per-node-heartbeat.maximum-container-assignments"; + /** + * Avoid potential risk that greedy assign multiple may involve + * */ @Private - public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1; + public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = 100; /** Configuring absolute min/max resources in a queue. **/ @Private - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/3] hadoop git commit: YARN-8456. Fix a configuration handling bug when user leave FPGA discover executable path configuration default but set OpenCL SDK path environment variable. (Zhankun Tang via
YARN-8456. Fix a configuration handling bug when user leave FPGA discover
executable path configuration default but set OpenCL SDK path environment
variable. (Zhankun Tang via wangda)
Change-Id: Iff150ea98ba0c60d448474fd940eb121afce6965
(cherry picked from commit a457a8951a1b35f06811c40443ca44bb9c698c30)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f7ed043
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f7ed043
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f7ed043
Branch: refs/heads/branch-3.2
Commit: 5f7ed043c8e8f9d964a719d076a9d3d207d4cea7
Parents: a7785d2
Author: Wangda Tan
Authored: Thu Oct 18 10:57:11 2018 -0700
Committer: Wangda Tan
Committed: Thu Oct 18 12:12:14 2018 -0700
--
.../fpga/IntelFpgaOpenclPlugin.java | 4 +-
.../resourceplugin/fpga/TestFpgaDiscoverer.java | 64 ++--
2 files changed, 61 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f7ed043/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java
index 2d6cf6f..f8fb6d8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java
@@ -124,7 +124,9 @@ public class IntelFpgaOpenclPlugin implements
AbstractFpgaVendorPlugin {
} else {
binaryPath = new File(pluginDefaultPreferredPath + "/bin",
pluginDefaultBinaryName);
if (binaryPath.exists()) {
- pathToExecutable = pluginDefaultPreferredPath;
+ pathToExecutable = binaryPath.getAbsolutePath();
+ LOG.info("Succeed in finding FPGA discoverer executable: " +
+ pathToExecutable);
} else {
pathToExecutable = pluginDefaultBinaryName;
LOG.warn("Failed to find FPGA discoverer executable in " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f7ed043/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java
index 87fb4e9..d5bcdb3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java
@@ -32,9 +32,8 @@ import org.junit.Test;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import java.lang.reflect.Field;
+import java.util.*;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.anyString;
@@ -60,8 +59,41 @@ public class TestFpgaDiscoverer {
f.mkdirs();
}
+ // A dirty hack to modify the env of the current JVM itself - Dirty, but
+ // should be okay for testing.
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private static void setNewEnvironmentHack(Map newenv)
+ throws Exception {
+try {
+ Class cl = Class.forName("java.lang.ProcessEnvironment");
+ Field field = cl.getDeclaredField("theEnvironment"
[1/3] hadoop git commit: YARN-6098. Add documentation for Delete Queue. (Suma Shivaprasad via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.2 4681e17a3 -> 7252f8e11 YARN-6098. Add documentation for Delete Queue. (Suma Shivaprasad via wangda) Change-Id: I23bf7ca0acbe1168a7f805ceccdd62fe41bfeb35 (cherry picked from commit bfb88b10f46a265aa38ab3e1d87b6a0a99d94be8) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a7785d27 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a7785d27 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a7785d27 Branch: refs/heads/branch-3.2 Commit: a7785d2764767ffed7acb0cef7d2894c9a43618a Parents: 4681e17 Author: Wangda Tan Authored: Thu Oct 18 10:55:39 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 12:12:09 2018 -0700 -- .../src/site/markdown/CapacityScheduler.md| 14 +- 1 file changed, 13 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7785d27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 5ac1d0a..c2d4ce4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -54,7 +54,7 @@ The `CapacityScheduler` supports the following features: * **Operability** -* Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime. +* Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime unless the queue is STOPPED and nhas no pending/running apps. * Drain applications - Administrators can *stop* queues at runtime to ensure that while existing applications run to completion, no new applications can be submitted. If a queue is in `STOPPED` state, new applications cannot be submitted to *itself* or *any of its child queues*. Existing applications continue to completion, thus the queue can be *drained* gracefully. Administrators can also *start* the stopped queues. @@ -441,6 +441,18 @@ Changing queue/scheduler properties and adding/removing queues can be done in tw $ vi $HADOOP_CONF_DIR/capacity-scheduler.xml $ $HADOOP_YARN_HOME/bin/yarn rmadmin -refreshQueues + Deleting queue via file + + Step 1: Stop the queue + + Before deleting a leaf queue, the leaf queue should not have any running/pending apps and has to BE STOPPED by changing `yarn.scheduler.capacity..state`. See the + [Queue Administration & Permissions](CapacityScheduler.html#Queue Properties) section. + Before deleting a parent queue, all its child queues should not have any running/pending apps and have to BE STOPPED. The parent queue also needs to be STOPPED + + Step 2: Delete the queue + + Remove the queue configurations from the file and run refresh as described above + ### Changing queue configuration via API Editing by API uses a backing store for the scheduler configuration. To enable this, the following parameters can be configured in yarn-site.xml. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[3/3] hadoop git commit: YARN-8896. Limit the maximum number of container assignments per heartbeat. (Zhankun Tang via wangda)
YARN-8896. Limit the maximum number of container assignments per heartbeat. (Zhankun Tang via wangda) Change-Id: I6e72f8362bd7f5c2a844cb9e3c4732492314e9f1 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/780be14f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/780be14f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/780be14f Branch: refs/heads/trunk Commit: 780be14f07df2a3ed6273b96ae857c278fd72718 Parents: a457a89 Author: Wangda Tan Authored: Thu Oct 18 10:58:21 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 10:58:21 2018 -0700 -- .../scheduler/capacity/CapacitySchedulerConfiguration.java | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/780be14f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index b937ae7..08380f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -336,8 +336,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public static final String MAX_ASSIGN_PER_HEARTBEAT = PREFIX + "per-node-heartbeat.maximum-container-assignments"; + /** + * Avoid potential risk that greedy assign multiple may involve + * */ @Private - public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1; + public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = 100; /** Configuring absolute min/max resources in a queue. **/ @Private - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/3] hadoop git commit: YARN-6098. Add documentation for Delete Queue. (Suma Shivaprasad via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 0e56c883c -> 780be14f0 YARN-6098. Add documentation for Delete Queue. (Suma Shivaprasad via wangda) Change-Id: I23bf7ca0acbe1168a7f805ceccdd62fe41bfeb35 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bfb88b10 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bfb88b10 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bfb88b10 Branch: refs/heads/trunk Commit: bfb88b10f46a265aa38ab3e1d87b6a0a99d94be8 Parents: 0e56c88 Author: Wangda Tan Authored: Thu Oct 18 10:55:39 2018 -0700 Committer: Wangda Tan Committed: Thu Oct 18 10:55:39 2018 -0700 -- .../src/site/markdown/CapacityScheduler.md| 14 +- 1 file changed, 13 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb88b10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md index 5ac1d0a..c2d4ce4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md @@ -54,7 +54,7 @@ The `CapacityScheduler` supports the following features: * **Operability** -* Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime. +* Runtime Configuration - The queue definitions and properties such as capacity, ACLs can be changed, at runtime, by administrators in a secure manner to minimize disruption to users. Also, a console is provided for users and administrators to view current allocation of resources to various queues in the system. Administrators can *add additional queues* at runtime, but queues cannot be *deleted* at runtime unless the queue is STOPPED and nhas no pending/running apps. * Drain applications - Administrators can *stop* queues at runtime to ensure that while existing applications run to completion, no new applications can be submitted. If a queue is in `STOPPED` state, new applications cannot be submitted to *itself* or *any of its child queues*. Existing applications continue to completion, thus the queue can be *drained* gracefully. Administrators can also *start* the stopped queues. @@ -441,6 +441,18 @@ Changing queue/scheduler properties and adding/removing queues can be done in tw $ vi $HADOOP_CONF_DIR/capacity-scheduler.xml $ $HADOOP_YARN_HOME/bin/yarn rmadmin -refreshQueues + Deleting queue via file + + Step 1: Stop the queue + + Before deleting a leaf queue, the leaf queue should not have any running/pending apps and has to BE STOPPED by changing `yarn.scheduler.capacity..state`. See the + [Queue Administration & Permissions](CapacityScheduler.html#Queue Properties) section. + Before deleting a parent queue, all its child queues should not have any running/pending apps and have to BE STOPPED. The parent queue also needs to be STOPPED + + Step 2: Delete the queue + + Remove the queue configurations from the file and run refresh as described above + ### Changing queue configuration via API Editing by API uses a backing store for the scheduler configuration. To enable this, the following parameters can be configured in yarn-site.xml. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.1 30526f2c5 -> 3e1894c37 YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda) Change-Id: I8bc8622936861b8d6de3e42a0b75af86ad8a3961 (cherry picked from commit 538250db26ce0b261bb74053348cddfc2d65cf52) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e1894c3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e1894c3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e1894c3 Branch: refs/heads/branch-3.1 Commit: 3e1894c3766e42d6756758d0d90d97eefc2ef331 Parents: 30526f2 Author: Wangda Tan Authored: Tue Oct 16 13:41:17 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 14:36:36 2018 -0700 -- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e1894c3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md index 609ebe1..4c9daed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md @@ -52,4 +52,4 @@ Open your browser, go to `rm-address:8088/ui2` and try it! Notes - -- This UI framework is not verified under security environment, please use with caution under security environment. +This UI framework is verified under security environment as well. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[4/5] hadoop git commit: YARN-8798. [Submarine] Job should not be submitted if --input_path option is missing. (Zhankun Tang via wangda)
YARN-8798. [Submarine] Job should not be submitted if --input_path option is
missing. (Zhankun Tang via wangda)
Change-Id: I7ae0e44eb5179b04a6ac861ec1c65f3b18c38f0f
(cherry picked from commit 143d74775b2b62884090fdd88874134b9eab2888)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa23c8d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa23c8d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa23c8d7
Branch: refs/heads/branch-3.2
Commit: aa23c8d7396430f3863d56e2ea0d94ac3fdb34a0
Parents: 920401b
Author: Wangda Tan
Authored: Tue Oct 16 13:39:34 2018 -0700
Committer: Wangda Tan
Committed: Tue Oct 16 13:53:03 2018 -0700
--
.../client/cli/param/RunJobParameters.java | 6 +++
.../client/cli/TestRunJobCliParsing.java| 39
2 files changed, 45 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa23c8d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
index 92a1883..d923e0f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
@@ -62,6 +62,12 @@ public class RunJobParameters extends RunParameters {
if (parsedCommandLine.getOptionValue(CliConstants.N_WORKERS) != null) {
nWorkers = Integer.parseInt(
parsedCommandLine.getOptionValue(CliConstants.N_WORKERS));
+ // Only check null value.
+ // Training job shouldn't ignore INPUT_PATH option
+ // But if nWorkers is 0, INPUT_PATH can be ignored because user can only
run Tensorboard
+ if (null == input && 0 != nWorkers) {
+throw new ParseException("\"--" + CliConstants.INPUT_PATH + "\" is
absent");
+ }
}
int nPS = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa23c8d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
index 295d6a8..240de06 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.submarine.client.cli;
+import org.apache.commons.cli.ParseException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
@@ -136,6 +137,44 @@ public class TestRunJobCliParsing {
}
@Test
+ public void testNoInputPathOptionSpecified() throws Exception {
+RunJobCli runJobCli = new RunJobCli(getMockClientContext());
+String expectedErrorMessage = "\"--" + CliConstants.INPUT_PATH + "\" is
absent";
+String actualMessage = "";
+try {
+ runJobCli.run(
+ new String[]{"--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--checkpoint_path", "hdfs://output",
+ "--num_workers", "1", "--worker_launch_cmd", "python run-job.py",
+ "--worker_resources", "memory=4g,vcores=2", "--tensorboard",
+ "true", "--verbose", "--wait_job_finish"});
+} catch (ParseException e) {
+
[2/5] hadoop git commit: YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/920401b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
new file mode 100644
index 000..7b3c2a9
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## @description check install user
+## @audience public
+## @stabilitystable
+function check_install_user()
+{
+ if [[ $(id -u) -ne 0 ]];then
+echo "This script must be run with a ROOT user!"
+exit # don't call exit_install()
+ fi
+}
+
+## @description exit install
+## @audience public
+## @stabilitystable
+function exit_install()
+{
+ echo "Exit the installation!" | tee -a $LOG
+ exit $1
+}
+
+## @description Check if the IP address format is correct
+## @audience public
+## @stabilitystable
+function valid_ip()
+{
+ local ip=$1
+ local stat=1
+
+ if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then
+OIFS=$IFS
+IFS='.'
+ip=($ip)
+IFS=$OIFS
+
+if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]}
-le 255 ]]; then
+ stat=$?
+fi
+ fi
+
+ return $stat
+}
+
+## @description Check if the configuration file configuration is correct
+## @audience public
+## @stabilitystable
+function check_install_conf()
+{
+ echo "Check if the configuration file configuration is correct ..." | tee -a
$LOG
+
+ # check etcd conf
+ hostCount=${#ETCD_HOSTS[@]}
+ if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2
+echo "Number of nodes = [$hostCount], must be configured to be greater
than or equal to 3 servers! " | tee -a $LOG
+exit_install
+ fi
+ for ip in ${ETCD_HOSTS[@]}
+ do
+if ! valid_ip $ip; then
+ echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG
+ exit_install
+fi
+ done
+ echo "Check if the configuration file configuration is correct [ Done ]" |
tee -a $LOG
+}
+
+## @description index by EtcdHosts list
+## @audience public
+## @stabilitystable
+function indexByEtcdHosts() {
+ index=0
+ while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do
+if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then
+ echo $index
+ return
+fi
+let "index++"
+ done
+ echo ""
+}
+
+## @description get local IP
+## @audience public
+## @stabilitystable
+function getLocalIP()
+{
+ local _ip _myip _line _nl=$'\n'
+ while IFS=$': \t' read -a _line ;do
+ [ -z "${_line%inet}" ] &&
+ _ip=${_line[${#_line[1]}>4?1:2]} &&
+ [ "${_ip#127.0.0.1}" ] && _myip=$_ip
+done< <(LANG=C /sbin/ifconfig)
+ printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip
+}
+
+## @description get ip list
+## @audience public
+## @stabilitystable
+function get_ip_list()
+{
+ array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[
\t]*//g' | cut -d ' ' -f2)
+
+ for ip in ${array[@]}
+ do
+LOCAL_HOST_IP_LIST+=(${ip})
+ done
+}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/5] hadoop git commit: YARN-8875. [Submarine] Add documentation for submarine installation script details. (Xun Liu via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.2 c19864d8c -> 80ee60c88 YARN-8875. [Submarine] Add documentation for submarine installation script details. (Xun Liu via wangda) Change-Id: I1c8d39c394e5a30f967ea514919835b951f2c124 (cherry picked from commit ed08dd3b0c9cec20373e8ca4e34d6526bd759943) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8610c207 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8610c207 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8610c207 Branch: refs/heads/branch-3.2 Commit: 8610c20724141563f34025ba8d67c8d2ed2c54e5 Parents: c19864d Author: Wangda Tan Authored: Tue Oct 16 13:36:09 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:52:53 2018 -0700 -- .../src/site/markdown/HowToInstall.md | 36 +++ .../src/site/markdown/Index.md | 4 +- .../src/site/markdown/InstallationGuide.md | 205 +++ .../src/site/markdown/InstallationScriptCN.md | 242 ++ .../src/site/markdown/InstallationScriptEN.md | 250 +++ .../src/site/markdown/TestAndTroubleshooting.md | 165 .../resources/images/submarine-installer.gif| Bin 0 -> 546547 bytes 7 files changed, 724 insertions(+), 178 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8610c207/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md new file mode 100644 index 000..05d87c1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md @@ -0,0 +1,36 @@ + + +# How to Install Dependencies + +Submarine project uses YARN Service, Docker container, and GPU (when GPU hardware available and properly configured). + +That means as an admin, you have to properly setup YARN Service related dependencies, including: +- YARN Registry DNS + +Docker related dependencies, including: +- Docker binary with expected versions. +- Docker network which allows Docker container can talk to each other across different nodes. + +And when GPU wanna to be used: +- GPU Driver. +- Nvidia-docker. + +For your convenience, we provided installation documents to help you to setup your environment. You can always choose to have them installed in your own way. + +Use Submarine installer to install dependencies: [EN](InstallationScriptEN.html) [CN](InstallationScriptCN.html) + +Alternatively, you can follow manual install dependencies: [EN](InstallationGuide.html) [CN](InstallationGuideChineseVersion.html) + +Once you have installed dependencies, please follow following guide to [TestAndTroubleshooting](TestAndTroubleshooting.html). \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/8610c207/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md index 0006f6c..baeaa15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md @@ -41,6 +41,4 @@ Click below contents if you want to understand more. - [Developer guide](DeveloperGuide.html) -- [Installation guide](InstallationGuide.html) - -- [Installation guide Chinese version](InstallationGuideChineseVersion.html) +- [Installation guides](HowToInstall.html) http://git-wip-us.apache.org/repos/asf/hadoop/blob/8610c207/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md index d4f4269..4ef2bda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applic
[5/5] hadoop git commit: YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda)
YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda) Change-Id: I8bc8622936861b8d6de3e42a0b75af86ad8a3961 (cherry picked from commit 538250db26ce0b261bb74053348cddfc2d65cf52) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80ee60c8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80ee60c8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80ee60c8 Branch: refs/heads/branch-3.2 Commit: 80ee60c88ca91381fecc6c2aaaf0804a39f38a8d Parents: aa23c8d Author: Wangda Tan Authored: Tue Oct 16 13:41:17 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:53:08 2018 -0700 -- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/80ee60c8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md index 609ebe1..4c9daed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md @@ -52,4 +52,4 @@ Open your browser, go to `rm-address:8088/ui2` and try it! Notes - -- This UI framework is not verified under security environment, please use with caution under security environment. +This UI framework is verified under security environment as well. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[3/5] hadoop git commit: YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)
YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda) Change-Id: I46e8d9fd32c7745c313030da62da41486a77b3ea (cherry picked from commit 46d6e0016610ced51a76189daeb3ad0e3dbbf94c) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/920401b2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/920401b2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/920401b2 Branch: refs/heads/branch-3.2 Commit: 920401b2966c2dcd7cd1b3dfd7a764f08412323f Parents: 8610c20 Author: Wangda Tan Authored: Tue Oct 16 13:36:59 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:52:58 2018 -0700 -- .../resources/assemblies/hadoop-yarn-dist.xml | 8 + .../installation/install.conf | 74 .../installation/install.sh | 116 + .../package/calico/calico-node.service | 50 +++ .../installation/package/calico/calicoctl.cfg | 22 + .../installation/package/docker/daemon.json | 23 + .../installation/package/docker/docker.service | 35 ++ .../installation/package/etcd/etcd.service | 40 ++ .../package/hadoop/container-executor.cfg | 41 ++ .../installation/package/submarine/submarine.sh | 25 ++ .../installation/scripts/calico.sh | 224 ++ .../installation/scripts/docker.sh | 166 +++ .../installation/scripts/download-server.sh | 42 ++ .../installation/scripts/environment.sh | 213 + .../installation/scripts/etcd.sh| 152 +++ .../installation/scripts/hadoop.sh | 117 + .../installation/scripts/menu.sh| 444 +++ .../installation/scripts/nvidia-docker.sh | 99 + .../installation/scripts/nvidia.sh | 120 + .../installation/scripts/submarine.sh | 38 ++ .../installation/scripts/utils.sh | 123 + 21 files changed, 2172 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/920401b2/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index a2ea08c..9c401e8 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -66,6 +66,14 @@ 0755 + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation + /share/hadoop/yarn/submarine-installer + +**/* + + 0755 + + hadoop-yarn/conf etc/hadoop http://git-wip-us.apache.org/repos/asf/hadoop/blob/920401b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf new file mode 100644 index 000..82dcf61 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# install config # + +# DNS +LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf +YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address + +# etcd hosts list +ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) + +# docker registry ip:port +DOCKER_REGISTRY="10.120.196.232:5000" + +# Start the http download service on the specified server, +# Will download all the dependencies in the http server, +# Run the install script on other servers. +# Automatically download dependencies from http, +# Solve
[2/5] hadoop git commit: YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
new file mode 100644
index 000..7b3c2a9
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## @description check install user
+## @audience public
+## @stabilitystable
+function check_install_user()
+{
+ if [[ $(id -u) -ne 0 ]];then
+echo "This script must be run with a ROOT user!"
+exit # don't call exit_install()
+ fi
+}
+
+## @description exit install
+## @audience public
+## @stabilitystable
+function exit_install()
+{
+ echo "Exit the installation!" | tee -a $LOG
+ exit $1
+}
+
+## @description Check if the IP address format is correct
+## @audience public
+## @stabilitystable
+function valid_ip()
+{
+ local ip=$1
+ local stat=1
+
+ if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then
+OIFS=$IFS
+IFS='.'
+ip=($ip)
+IFS=$OIFS
+
+if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]}
-le 255 ]]; then
+ stat=$?
+fi
+ fi
+
+ return $stat
+}
+
+## @description Check if the configuration file configuration is correct
+## @audience public
+## @stabilitystable
+function check_install_conf()
+{
+ echo "Check if the configuration file configuration is correct ..." | tee -a
$LOG
+
+ # check etcd conf
+ hostCount=${#ETCD_HOSTS[@]}
+ if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2
+echo "Number of nodes = [$hostCount], must be configured to be greater
than or equal to 3 servers! " | tee -a $LOG
+exit_install
+ fi
+ for ip in ${ETCD_HOSTS[@]}
+ do
+if ! valid_ip $ip; then
+ echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG
+ exit_install
+fi
+ done
+ echo "Check if the configuration file configuration is correct [ Done ]" |
tee -a $LOG
+}
+
+## @description index by EtcdHosts list
+## @audience public
+## @stabilitystable
+function indexByEtcdHosts() {
+ index=0
+ while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do
+if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then
+ echo $index
+ return
+fi
+let "index++"
+ done
+ echo ""
+}
+
+## @description get local IP
+## @audience public
+## @stabilitystable
+function getLocalIP()
+{
+ local _ip _myip _line _nl=$'\n'
+ while IFS=$': \t' read -a _line ;do
+ [ -z "${_line%inet}" ] &&
+ _ip=${_line[${#_line[1]}>4?1:2]} &&
+ [ "${_ip#127.0.0.1}" ] && _myip=$_ip
+done< <(LANG=C /sbin/ifconfig)
+ printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip
+}
+
+## @description get ip list
+## @audience public
+## @stabilitystable
+function get_ip_list()
+{
+ array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[
\t]*//g' | cut -d ' ' -f2)
+
+ for ip in ${array[@]}
+ do
+LOCAL_HOST_IP_LIST+=(${ip})
+ done
+}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[5/5] hadoop git commit: YARN-8798. [Submarine] Job should not be submitted if --input_path option is missing. (Zhankun Tang via wangda)
YARN-8798. [Submarine] Job should not be submitted if --input_path option is
missing. (Zhankun Tang via wangda)
Change-Id: I7ae0e44eb5179b04a6ac861ec1c65f3b18c38f0f
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/143d7477
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/143d7477
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/143d7477
Branch: refs/heads/trunk
Commit: 143d74775b2b62884090fdd88874134b9eab2888
Parents: 46d6e00
Author: Wangda Tan
Authored: Tue Oct 16 13:39:34 2018 -0700
Committer: Wangda Tan
Committed: Tue Oct 16 13:51:02 2018 -0700
--
.../client/cli/param/RunJobParameters.java | 6 +++
.../client/cli/TestRunJobCliParsing.java| 39
2 files changed, 45 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143d7477/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
index 92a1883..d923e0f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
@@ -62,6 +62,12 @@ public class RunJobParameters extends RunParameters {
if (parsedCommandLine.getOptionValue(CliConstants.N_WORKERS) != null) {
nWorkers = Integer.parseInt(
parsedCommandLine.getOptionValue(CliConstants.N_WORKERS));
+ // Only check null value.
+ // Training job shouldn't ignore INPUT_PATH option
+ // But if nWorkers is 0, INPUT_PATH can be ignored because user can only
run Tensorboard
+ if (null == input && 0 != nWorkers) {
+throw new ParseException("\"--" + CliConstants.INPUT_PATH + "\" is
absent");
+ }
}
int nPS = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143d7477/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
index 295d6a8..240de06 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.submarine.client.cli;
+import org.apache.commons.cli.ParseException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
@@ -136,6 +137,44 @@ public class TestRunJobCliParsing {
}
@Test
+ public void testNoInputPathOptionSpecified() throws Exception {
+RunJobCli runJobCli = new RunJobCli(getMockClientContext());
+String expectedErrorMessage = "\"--" + CliConstants.INPUT_PATH + "\" is
absent";
+String actualMessage = "";
+try {
+ runJobCli.run(
+ new String[]{"--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--checkpoint_path", "hdfs://output",
+ "--num_workers", "1", "--worker_launch_cmd", "python run-job.py",
+ "--worker_resources", "memory=4g,vcores=2", "--tensorboard",
+ "true", "--verbose", "--wait_job_finish"});
+} catch (ParseException e) {
+ actualMessage = e.getMessage();
+ e.printStackTrace();
+}
+Assert.assertEquals(expectedE
[3/5] hadoop git commit: YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)
YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda) Change-Id: I46e8d9fd32c7745c313030da62da41486a77b3ea Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46d6e001 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46d6e001 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46d6e001 Branch: refs/heads/trunk Commit: 46d6e0016610ced51a76189daeb3ad0e3dbbf94c Parents: ed08dd3 Author: Wangda Tan Authored: Tue Oct 16 13:36:59 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:51:02 2018 -0700 -- .../resources/assemblies/hadoop-yarn-dist.xml | 8 + .../installation/install.conf | 74 .../installation/install.sh | 116 + .../package/calico/calico-node.service | 50 +++ .../installation/package/calico/calicoctl.cfg | 22 + .../installation/package/docker/daemon.json | 23 + .../installation/package/docker/docker.service | 35 ++ .../installation/package/etcd/etcd.service | 40 ++ .../package/hadoop/container-executor.cfg | 41 ++ .../installation/package/submarine/submarine.sh | 25 ++ .../installation/scripts/calico.sh | 224 ++ .../installation/scripts/docker.sh | 166 +++ .../installation/scripts/download-server.sh | 42 ++ .../installation/scripts/environment.sh | 213 + .../installation/scripts/etcd.sh| 152 +++ .../installation/scripts/hadoop.sh | 117 + .../installation/scripts/menu.sh| 444 +++ .../installation/scripts/nvidia-docker.sh | 99 + .../installation/scripts/nvidia.sh | 120 + .../installation/scripts/submarine.sh | 38 ++ .../installation/scripts/utils.sh | 123 + 21 files changed, 2172 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index a2ea08c..9c401e8 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -66,6 +66,14 @@ 0755 + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation + /share/hadoop/yarn/submarine-installer + +**/* + + 0755 + + hadoop-yarn/conf etc/hadoop http://git-wip-us.apache.org/repos/asf/hadoop/blob/46d6e001/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf new file mode 100644 index 000..82dcf61 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# install config # + +# DNS +LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf +YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address + +# etcd hosts list +ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) + +# docker registry ip:port +DOCKER_REGISTRY="10.120.196.232:5000" + +# Start the http download service on the specified server, +# Will download all the dependencies in the http server, +# Run the install script on other servers. +# Automatically download dependencies from http, +# Solve the problem that all servers are slow to download online. +# At the same tim
[4/5] hadoop git commit: YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda)
YARN-8892. YARN UI2 doc changes to update security status (verified under security environment). (Sunil G via wangda) Change-Id: I8bc8622936861b8d6de3e42a0b75af86ad8a3961 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/538250db Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/538250db Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/538250db Branch: refs/heads/trunk Commit: 538250db26ce0b261bb74053348cddfc2d65cf52 Parents: 143d747 Author: Wangda Tan Authored: Tue Oct 16 13:41:17 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:51:02 2018 -0700 -- .../hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/538250db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md index 609ebe1..4c9daed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnUI2.md @@ -52,4 +52,4 @@ Open your browser, go to `rm-address:8088/ui2` and try it! Notes - -- This UI framework is not verified under security environment, please use with caution under security environment. +This UI framework is verified under security environment as well. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/5] hadoop git commit: YARN-8875. [Submarine] Add documentation for submarine installation script details. (Xun Liu via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk babd1449b -> 538250db2 YARN-8875. [Submarine] Add documentation for submarine installation script details. (Xun Liu via wangda) Change-Id: I1c8d39c394e5a30f967ea514919835b951f2c124 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed08dd3b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed08dd3b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed08dd3b Branch: refs/heads/trunk Commit: ed08dd3b0c9cec20373e8ca4e34d6526bd759943 Parents: babd144 Author: Wangda Tan Authored: Tue Oct 16 13:36:09 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 16 13:51:01 2018 -0700 -- .../src/site/markdown/HowToInstall.md | 36 +++ .../src/site/markdown/Index.md | 4 +- .../src/site/markdown/InstallationGuide.md | 205 +++ .../src/site/markdown/InstallationScriptCN.md | 242 ++ .../src/site/markdown/InstallationScriptEN.md | 250 +++ .../src/site/markdown/TestAndTroubleshooting.md | 165 .../resources/images/submarine-installer.gif| Bin 0 -> 546547 bytes 7 files changed, 724 insertions(+), 178 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed08dd3b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md new file mode 100644 index 000..05d87c1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/HowToInstall.md @@ -0,0 +1,36 @@ + + +# How to Install Dependencies + +Submarine project uses YARN Service, Docker container, and GPU (when GPU hardware available and properly configured). + +That means as an admin, you have to properly setup YARN Service related dependencies, including: +- YARN Registry DNS + +Docker related dependencies, including: +- Docker binary with expected versions. +- Docker network which allows Docker container can talk to each other across different nodes. + +And when GPU wanna to be used: +- GPU Driver. +- Nvidia-docker. + +For your convenience, we provided installation documents to help you to setup your environment. You can always choose to have them installed in your own way. + +Use Submarine installer to install dependencies: [EN](InstallationScriptEN.html) [CN](InstallationScriptCN.html) + +Alternatively, you can follow manual install dependencies: [EN](InstallationGuide.html) [CN](InstallationGuideChineseVersion.html) + +Once you have installed dependencies, please follow following guide to [TestAndTroubleshooting](TestAndTroubleshooting.html). \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed08dd3b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md index 0006f6c..baeaa15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md @@ -41,6 +41,4 @@ Click below contents if you want to understand more. - [Developer guide](DeveloperGuide.html) -- [Installation guide](InstallationGuide.html) - -- [Installation guide Chinese version](InstallationGuideChineseVersion.html) +- [Installation guides](HowToInstall.html) http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed08dd3b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md index d4f4269..4ef2bda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md @@ -16,9
hadoop git commit: YARN-8852. Add documentation for submarine installation details. (Zac Zhou via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-3.2 f6a73d181 -> 8e8b74872 YARN-8852. Add documentation for submarine installation details. (Zac Zhou via wangda) Change-Id: If5681d1ef37ff5dc916735eeef15a6120173d653 (cherry picked from commit a23ea68b9747eae9b176f908bb04b76d30fe3795) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e8b7487 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e8b7487 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e8b7487 Branch: refs/heads/branch-3.2 Commit: 8e8b74872498077a1c3568b5f55b4e215669bad4 Parents: f6a73d1 Author: Wangda Tan Authored: Tue Oct 9 10:18:00 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 9 10:19:16 2018 -0700 -- .../src/site/markdown/Index.md | 4 + .../src/site/markdown/InstallationGuide.md | 760 +++ .../markdown/InstallationGuideChineseVersion.md | 757 ++ 3 files changed, 1521 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e8b7487/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md index 0b78a87..0006f6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md @@ -40,3 +40,7 @@ Click below contents if you want to understand more. - [How to write Dockerfile for Submarine jobs](WriteDockerfile.html) - [Developer guide](DeveloperGuide.html) + +- [Installation guide](InstallationGuide.html) + +- [Installation guide Chinese version](InstallationGuideChineseVersion.html) http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e8b7487/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md new file mode 100644 index 000..d4f4269 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md @@ -0,0 +1,760 @@ + + +# Submarine Installation Guide + +## Prerequisites + +### Operating System + +The operating system and kernel versions we used are as shown in the following table, which should be minimum required versions: + +| Enviroment | Verion | +| -- | -- | +| Operating System | centos-release-7-3.1611.el7.centos.x86_64 | +| Kernal | 3.10.0-514.el7.x86_64 | + +### User & Group + +As there are some specific users and groups need to be created to install hadoop/docker. Please create them if they are missing. + +``` +adduser hdfs +adduser mapred +adduser yarn +addgroup hadoop +usermod -aG hdfs,hadoop hdfs +usermod -aG mapred,hadoop mapred +usermod -aG yarn,hadoop yarn +usermod -aG hdfs,hadoop hadoop +groupadd docker +usermod -aG docker yarn +usermod -aG docker hadoop +``` + +### GCC Version + +Check the version of GCC tool + +```bash +gcc --version +gcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-11) +# install if needed +yum install gcc make g++ +``` + +### Kernel header & Kernel devel + +```bash +# Approach 1ï¼ +yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r) +# Approach 2ï¼ +wget http://vault.centos.org/7.3.1611/os/x86_64/Packages/kernel-headers-3.10.0-514.el7.x86_64.rpm +rpm -ivh kernel-headers-3.10.0-514.el7.x86_64.rpm +``` + +### GPU Servers + +``` +lspci | grep -i nvidia + +# If the server has gpus, you can get info like thisï¼ +04:00.0 3D controller: NVIDIA Corporation Device 1b38 (rev a1) +82:00.0 3D controller: NVIDIA Corporation Device 1b38 (rev a1) +``` + + + +### Nvidia Driver Installation + +If nvidia driver/cuda has been installed before, They should be uninstalled firstly. + +``` +# uninstall cudaï¼ +sudo /usr/local/cuda-10.0/bin/uninstall_cuda_10.0.pl + +# uninstall nvidia-driverï¼ +sudo /usr/bin/nvidia-uninstall +``` + +To check GPU version, install nvidia-detect + +``` +yum install nvidia-detect +# run 'nvidia-detect -v' to get reqired nvidia driver versionï¼ +nvidia-detect -v +Probing for supported NVIDIA devices... +[10de:13bb] NVIDIA Corporation GM107GL [Quadro K620] +This device requires the cur
hadoop git commit: YARN-8852. Add documentation for submarine installation details. (Zac Zhou via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 600438bcd -> a23ea68b9 YARN-8852. Add documentation for submarine installation details. (Zac Zhou via wangda) Change-Id: If5681d1ef37ff5dc916735eeef15a6120173d653 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a23ea68b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a23ea68b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a23ea68b Branch: refs/heads/trunk Commit: a23ea68b9747eae9b176f908bb04b76d30fe3795 Parents: 600438b Author: Wangda Tan Authored: Tue Oct 9 10:18:00 2018 -0700 Committer: Wangda Tan Committed: Tue Oct 9 10:18:27 2018 -0700 -- .../src/site/markdown/Index.md | 4 + .../src/site/markdown/InstallationGuide.md | 760 +++ .../markdown/InstallationGuideChineseVersion.md | 757 ++ 3 files changed, 1521 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ea68b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md index 0b78a87..0006f6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/Index.md @@ -40,3 +40,7 @@ Click below contents if you want to understand more. - [How to write Dockerfile for Submarine jobs](WriteDockerfile.html) - [Developer guide](DeveloperGuide.html) + +- [Installation guide](InstallationGuide.html) + +- [Installation guide Chinese version](InstallationGuideChineseVersion.html) http://git-wip-us.apache.org/repos/asf/hadoop/blob/a23ea68b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md new file mode 100644 index 000..d4f4269 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/site/markdown/InstallationGuide.md @@ -0,0 +1,760 @@ + + +# Submarine Installation Guide + +## Prerequisites + +### Operating System + +The operating system and kernel versions we used are as shown in the following table, which should be minimum required versions: + +| Enviroment | Verion | +| -- | -- | +| Operating System | centos-release-7-3.1611.el7.centos.x86_64 | +| Kernal | 3.10.0-514.el7.x86_64 | + +### User & Group + +As there are some specific users and groups need to be created to install hadoop/docker. Please create them if they are missing. + +``` +adduser hdfs +adduser mapred +adduser yarn +addgroup hadoop +usermod -aG hdfs,hadoop hdfs +usermod -aG mapred,hadoop mapred +usermod -aG yarn,hadoop yarn +usermod -aG hdfs,hadoop hadoop +groupadd docker +usermod -aG docker yarn +usermod -aG docker hadoop +``` + +### GCC Version + +Check the version of GCC tool + +```bash +gcc --version +gcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-11) +# install if needed +yum install gcc make g++ +``` + +### Kernel header & Kernel devel + +```bash +# Approach 1ï¼ +yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r) +# Approach 2ï¼ +wget http://vault.centos.org/7.3.1611/os/x86_64/Packages/kernel-headers-3.10.0-514.el7.x86_64.rpm +rpm -ivh kernel-headers-3.10.0-514.el7.x86_64.rpm +``` + +### GPU Servers + +``` +lspci | grep -i nvidia + +# If the server has gpus, you can get info like thisï¼ +04:00.0 3D controller: NVIDIA Corporation Device 1b38 (rev a1) +82:00.0 3D controller: NVIDIA Corporation Device 1b38 (rev a1) +``` + + + +### Nvidia Driver Installation + +If nvidia driver/cuda has been installed before, They should be uninstalled firstly. + +``` +# uninstall cudaï¼ +sudo /usr/local/cuda-10.0/bin/uninstall_cuda_10.0.pl + +# uninstall nvidia-driverï¼ +sudo /usr/bin/nvidia-uninstall +``` + +To check GPU version, install nvidia-detect + +``` +yum install nvidia-detect +# run 'nvidia-detect -v' to get reqired nvidia driver versionï¼ +nvidia-detect -v +Probing for supported NVIDIA devices... +[10de:13bb] NVIDIA Corporation GM107GL [Quadro K620] +This device requires the current 390.87 NVIDIA driver kmod-nvidia +[8086:1912] Intel Corporation HD Graphics 53
hadoop git commit: YARN-8844. TestNMProxy unit test is failing. (Eric Yang via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 6926fd0ec -> 2e9913caf
YARN-8844. TestNMProxy unit test is failing. (Eric Yang via wangda)
Change-Id: I241fa8701b6f1dbcad87fd2e9a429e32e7aa40f5
(cherry picked from commit b3ac8869338faeabedbc27e95e3166c830a9a761)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e9913ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e9913ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e9913ca
Branch: refs/heads/trunk
Commit: 2e9913caf2a0ba24c323c26dd978492bd398f2e5
Parents: 6926fd0
Author: Wangda Tan
Authored: Thu Oct 4 10:48:47 2018 -0700
Committer: Wangda Tan
Committed: Thu Oct 4 10:53:39 2018 -0700
--
.../yarn/server/nodemanager/containermanager/TestNMProxy.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e9913ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
index e6a7a02..5f023f0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
@@ -161,7 +161,7 @@ public class TestNMProxy extends BaseContainerManagerTest {
IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 100);
// connect to some dummy address so that it can trigger
// connection failure and RPC level retires.
-newConf.set(YarnConfiguration.NM_ADDRESS, "1234");
+newConf.set(YarnConfiguration.NM_ADDRESS, "0.0.0.0:1234");
ContainerManagementProtocol proxy = getNMProxy(newConf);
try {
proxy.startContainers(allRequests);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: YARN-8844. TestNMProxy unit test is failing. (Eric Yang via wangda)
YARN-8844. TestNMProxy unit test is failing. (Eric Yang via wangda)
Change-Id: I241fa8701b6f1dbcad87fd2e9a429e32e7aa40f5
(cherry picked from commit b3ac8869338faeabedbc27e95e3166c830a9a761)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86a1ad44
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86a1ad44
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86a1ad44
Branch: refs/heads/branch-3.1
Commit: 86a1ad442855cfa350c8bf791e85b09ddbecd0bd
Parents: 30f50a0
Author: Wangda Tan
Authored: Thu Oct 4 10:48:47 2018 -0700
Committer: Wangda Tan
Committed: Thu Oct 4 10:49:29 2018 -0700
--
.../yarn/server/nodemanager/containermanager/TestNMProxy.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a1ad44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
index e6a7a02..5f023f0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
@@ -161,7 +161,7 @@ public class TestNMProxy extends BaseContainerManagerTest {
IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 100);
// connect to some dummy address so that it can trigger
// connection failure and RPC level retires.
-newConf.set(YarnConfiguration.NM_ADDRESS, "1234");
+newConf.set(YarnConfiguration.NM_ADDRESS, "0.0.0.0:1234");
ContainerManagementProtocol proxy = getNMProxy(newConf);
try {
proxy.startContainers(allRequests);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/2] hadoop git commit: YARN-8758. Support getting PreemptionMessage when using AMRMClientAsyn. (Zian Chen via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 1a890b17b -> 86a1ad442
YARN-8758. Support getting PreemptionMessage when using AMRMClientAsyn. (Zian
Chen via wangda)
Change-Id: Ibf5d165f49957b582eeadeb41dc285c84d2f05e7
(cherry picked from commit 6926fd0ec634df2576bbc9f45e9636b99260db72)
(cherry picked from commit abe4a8e5d82e70ce991e2830f20eba9f25a2491a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30f50a06
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30f50a06
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30f50a06
Branch: refs/heads/branch-3.1
Commit: 30f50a0682d3ddb682a15ccf20f7fb797c17d18a
Parents: 1a890b1
Author: Wangda Tan
Authored: Thu Oct 4 10:31:33 2018 -0700
Committer: Wangda Tan
Committed: Thu Oct 4 10:49:23 2018 -0700
--
.../hadoop/yarn/client/api/async/AMRMClientAsync.java| 11 +++
.../yarn/client/api/async/impl/AMRMClientAsyncImpl.java | 9 +
2 files changed, 20 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30f50a06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
index 3dd2f71..c9f4e5f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -553,6 +554,16 @@ extends AbstractService {
public void onRequestsRejected(
List rejectedSchedulingRequests) {
}
+
+/**
+ * Called when the RM responds to a heartbeat with preemption message
+ * @param preemptionMessage
+ */
+@Public
+@Unstable
+public void onPreemptionMessageReceived(
+PreemptionMessage preemptionMessage) {
+}
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30f50a06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
index 3cf2c34..922b185 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -401,6 +402,14 @@ extends AMRMClientAsync {
handler.onContainersAllocated(allocated);
}
+ PreemptionMessage preemptionMessage =
response.getPreemptionMessage();
+ if (preemptionMessage != null) {
+if (handler instanceof AMRMClientAsync.AbstractCallbackHandler) {
+ ((AMRMClientAsync.AbstractCallbackHandler) handler)
+ .onPreemptionMessageReceived(preemptionMessage);
+}
+ }
+
if (!response.getContainersFromPreviousAttempts().isEmpty()) {
if (handler instanceof AMRMClientAsync.AbstractCallbackHandler) {
((AMRMClientAsync.AbstractCallbackHandler) hand
[2/2] hadoop git commit: YARN-8844. TestNMProxy unit test is failing. (Eric Yang via wangda)
YARN-8844. TestNMProxy unit test is failing. (Eric Yang via wangda)
Change-Id: I241fa8701b6f1dbcad87fd2e9a429e32e7aa40f5
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3ac8869
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3ac8869
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3ac8869
Branch: refs/heads/branch-3.2
Commit: b3ac8869338faeabedbc27e95e3166c830a9a761
Parents: abe4a8e
Author: Wangda Tan
Authored: Thu Oct 4 10:48:47 2018 -0700
Committer: Wangda Tan
Committed: Thu Oct 4 10:48:47 2018 -0700
--
.../yarn/server/nodemanager/containermanager/TestNMProxy.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3ac8869/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
index e6a7a02..5f023f0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
@@ -161,7 +161,7 @@ public class TestNMProxy extends BaseContainerManagerTest {
IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 100);
// connect to some dummy address so that it can trigger
// connection failure and RPC level retires.
-newConf.set(YarnConfiguration.NM_ADDRESS, "1234");
+newConf.set(YarnConfiguration.NM_ADDRESS, "0.0.0.0:1234");
ContainerManagementProtocol proxy = getNMProxy(newConf);
try {
proxy.startContainers(allRequests);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-8758. Support getting PreemptionMessage when using AMRMClientAsyn. (Zian Chen via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 81f635f47 -> 6926fd0ec
YARN-8758. Support getting PreemptionMessage when using AMRMClientAsyn. (Zian
Chen via wangda)
Change-Id: Ibf5d165f49957b582eeadeb41dc285c84d2f05e7
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6926fd0e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6926fd0e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6926fd0e
Branch: refs/heads/trunk
Commit: 6926fd0ec634df2576bbc9f45e9636b99260db72
Parents: 81f635f
Author: Wangda Tan
Authored: Thu Oct 4 10:31:33 2018 -0700
Committer: Wangda Tan
Committed: Thu Oct 4 10:31:33 2018 -0700
--
.../hadoop/yarn/client/api/async/AMRMClientAsync.java| 11 +++
.../yarn/client/api/async/impl/AMRMClientAsyncImpl.java | 9 +
2 files changed, 20 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6926fd0e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
index 3dd2f71..c9f4e5f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -553,6 +554,16 @@ extends AbstractService {
public void onRequestsRejected(
List rejectedSchedulingRequests) {
}
+
+/**
+ * Called when the RM responds to a heartbeat with preemption message
+ * @param preemptionMessage
+ */
+@Public
+@Unstable
+public void onPreemptionMessageReceived(
+PreemptionMessage preemptionMessage) {
+}
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6926fd0e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
index 3cf2c34..922b185 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -401,6 +402,14 @@ extends AMRMClientAsync {
handler.onContainersAllocated(allocated);
}
+ PreemptionMessage preemptionMessage =
response.getPreemptionMessage();
+ if (preemptionMessage != null) {
+if (handler instanceof AMRMClientAsync.AbstractCallbackHandler) {
+ ((AMRMClientAsync.AbstractCallbackHandler) handler)
+ .onPreemptionMessageReceived(preemptionMessage);
+}
+ }
+
if (!response.getContainersFromPreviousAttempts().isEmpty()) {
if (handler instanceof AMRMClientAsync.AbstractCallbackHandler) {
((AMRMClientAsync.AbstractCallbackHandler) handler)
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additio
hadoop git commit: MAPREDUCE-7125. JobResourceUploader creates LocalFileSystem when it's not necessary. (Peter Cseh via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 0196ffaa2 -> daa14bf06
MAPREDUCE-7125. JobResourceUploader creates LocalFileSystem when it's not
necessary. (Peter Cseh via wangda)
Change-Id: I1aa720ed03739f6f4abeec46f6068e2ab332987a
(cherry picked from commit 230d90c5b9cc0d6e74bf16b1cf6685a163a24d48)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/daa14bf0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/daa14bf0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/daa14bf0
Branch: refs/heads/branch-3.1
Commit: daa14bf06a56cc03513c4039c6da902af4d8b9d7
Parents: 0196ffa
Author: Wangda Tan
Authored: Mon Sep 24 13:30:02 2018 -0700
Committer: Wangda Tan
Committed: Mon Sep 24 13:53:51 2018 -0700
--
.../main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa14bf0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
index 1dbecb6..e106a54 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
@@ -855,8 +855,8 @@ class JobResourceUploader {
throw new IllegalArgumentException(e);
}
Path path = new Path(pathURI);
-FileSystem localFs = FileSystem.getLocal(conf);
if (pathURI.getScheme() == null) {
+ FileSystem localFs = FileSystem.getLocal(conf);
// default to the local file system
// check if the file exists or not first
localFs.getFileStatus(path);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: MAPREDUCE-7125. JobResourceUploader creates LocalFileSystem when it's not necessary. (Peter Cseh via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 67173b484 -> 230d90c5b
MAPREDUCE-7125. JobResourceUploader creates LocalFileSystem when it's not
necessary. (Peter Cseh via wangda)
Change-Id: I1aa720ed03739f6f4abeec46f6068e2ab332987a
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/230d90c5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/230d90c5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/230d90c5
Branch: refs/heads/trunk
Commit: 230d90c5b9cc0d6e74bf16b1cf6685a163a24d48
Parents: 67173b4
Author: Wangda Tan
Authored: Mon Sep 24 13:30:02 2018 -0700
Committer: Wangda Tan
Committed: Mon Sep 24 13:52:53 2018 -0700
--
.../main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/230d90c5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
index 1dbecb6..e106a54 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
@@ -855,8 +855,8 @@ class JobResourceUploader {
throw new IllegalArgumentException(e);
}
Path path = new Path(pathURI);
-FileSystem localFs = FileSystem.getLocal(conf);
if (pathURI.getScheme() == null) {
+ FileSystem localFs = FileSystem.getLocal(conf);
// default to the local file system
// check if the file exists or not first
localFs.getFileStatus(path);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-8698. [Submarine] Failed to reset Hadoop home environment when submitting a submarine job. (Zac Zhou via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 8944407d9 -> de85f841d
YARN-8698. [Submarine] Failed to reset Hadoop home environment when submitting
a submarine job. (Zac Zhou via wangda)
Change-Id: If7fdc11fcc4e49bed2186ded76cd69164f34f502
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de85f841
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de85f841
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de85f841
Branch: refs/heads/trunk
Commit: de85f841dae2bb22696472d78feb852883761a4e
Parents: 8944407
Author: Wangda Tan
Authored: Sun Sep 9 17:45:31 2018 -0700
Committer: Wangda Tan
Committed: Sun Sep 9 17:45:31 2018 -0700
--
.../runtimes/yarnservice/YarnServiceJobSubmitter.java| 11 +++
1 file changed, 7 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de85f841/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
index 3cd0d7e..a2a2067 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
@@ -125,6 +125,7 @@ public class YarnServiceJobSubmitter implements
JobSubmitter {
fw.append("export HADOOP_HOME=\n");
fw.append("export HADOOP_YARN_HOME=\n");
fw.append("export HADOOP_HDFS_HOME=" + hdfsHome + "\n");
+fw.append("export HADOOP_COMMON_HOME=" + hdfsHome + "\n");
} else{
lackingEnvs = true;
}
@@ -170,10 +171,12 @@ public class YarnServiceJobSubmitter implements
JobSubmitter {
// DEBUG
if (SubmarineLogs.isVerbose()) {
- fw.append("echo $CLASSPATH\n");
- fw.append("echo $JAVA_HOME\n");
- fw.append("echo $LD_LIBRARY_PATH\n");
- fw.append("echo $HADOOP_HDFS_HOME\n");
+ fw.append("echo \"CLASSPATH:$CLASSPATH\"\n");
+ fw.append("echo \"HADOOP_CONF_DIR:$HADOOP_CONF_DIR\"\n");
+ fw.append("echo
\"HADOOP_TOKEN_FILE_LOCATION:$HADOOP_TOKEN_FILE_LOCATION\"\n");
+ fw.append("echo \"JAVA_HOME:$JAVA_HOME\"\n");
+ fw.append("echo \"LD_LIBRARY_PATH:$LD_LIBRARY_PATH\"\n");
+ fw.append("echo \"HADOOP_HDFS_HOME:$HADOOP_HDFS_HOME\"\n");
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: YARN-8136. Add version attribute to site doc examples and quickstart. (Eric Yang via wangda)
YARN-8136. Add version attribute to site doc examples and quickstart. (Eric
Yang via wangda)
Change-Id: I4541b239f490ca0a6edf9698e0d3deaf83669151
(cherry picked from commit 8244abb7aeb768b73682b8c9a26516a9cf06bca5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/beea1873
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/beea1873
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/beea1873
Branch: refs/heads/branch-3.1
Commit: beea18736bbe6a81c2c76b0e904226146aa4dd3f
Parents: 68279fc
Author: Wangda Tan
Authored: Thu Aug 9 11:03:46 2018 -0700
Committer: Wangda Tan
Committed: Thu Aug 9 12:04:33 2018 -0700
--
.../hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md | 5 -
.../src/site/markdown/yarn-service/QuickStart.md| 1 +
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/beea1873/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
index 03fec79..73e00b3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
@@ -30,6 +30,7 @@ Note this example requires registry DNS.
```
{
"name": "httpd-service",
+ "version": "1.0",
"lifetime": "3600",
"components": [
{
@@ -169,9 +170,10 @@ Then visit port 8080 for each IP to view the pages.
Docker images may have built with ENTRYPOINT to enable start up of docker
image without any parameters.
When passing parameters to ENTRYPOINT enabled image, `launch_command` is
delimited by comma (,).
+```
{
"name": "sleeper-service",
- "version": "1",
+ "version": "1.0",
"components" :
[
{
@@ -198,3 +200,4 @@ When passing parameters to ENTRYPOINT enabled image,
`launch_command` is delimit
}
]
}
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/beea1873/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index e144320..bba9bb8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -40,6 +40,7 @@ Below is a simple service definition that launches sleep
containers on YARN by w
```
{
"name": "sleeper-service",
+ "version": "1.0",
"components" :
[
{
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/2] hadoop git commit: YARN-8588. Logging improvements for better debuggability. (Suma Shivaprasad via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 0ee7e8004 -> beea18736
YARN-8588. Logging improvements for better debuggability. (Suma Shivaprasad via
wangda)
Change-Id: I66aa4b0ec031ae5ce0fae558e2f8cbcbbfebc442
(cherry picked from commit 344c335a920e6f32a35ebace0a118a9dc4a22fb7)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68279fcd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68279fcd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68279fcd
Branch: refs/heads/branch-3.1
Commit: 68279fcd65f0c7c3eb8af1b0e922c615a97cbfb7
Parents: 0ee7e80
Author: Wangda Tan
Authored: Thu Aug 9 11:03:00 2018 -0700
Committer: Wangda Tan
Committed: Thu Aug 9 12:04:25 2018 -0700
--
.../capacity/AutoCreatedLeafQueueConfig.java| 5 ++
.../capacity/QueueManagementChange.java | 2 +-
.../QueueManagementDynamicEditPolicy.java | 36 ++
.../GuaranteedOrZeroCapacityOverTimePolicy.java | 50
4 files changed, 52 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68279fcd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
index 5952250..87ef1c0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
@@ -63,4 +63,9 @@ public class AutoCreatedLeafQueueConfig {
public CapacitySchedulerConfiguration getLeafQueueConfigs() {
return leafQueueConfigs;
}
+
+ @Override public String toString() {
+return "AutoCreatedLeafQueueConfig{" + "queueCapacities=" + queueCapacities
++ ", leafQueueConfigs=" + leafQueueConfigs + '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68279fcd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
index 74d9b23..64ba578 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
@@ -124,7 +124,7 @@ public abstract class QueueManagementChange {
@Override
public String toString() {
-return "QueueManagementChange{" + "queue=" + queue
+return "QueueManagementChange{" + "queue=" + queue.getQueueName()
+ ", updatedEntitlementsByPartition=" + queueTemplateUpdate
+ ", queueAction=" + queueAction + ", transitionToQueueState="
+ transitionToQueueState + '}';
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68279fcd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
[2/2] hadoop git commit: YARN-8588. Logging improvements for better debuggability. (Suma Shivaprasad via wangda)
YARN-8588. Logging improvements for better debuggability. (Suma Shivaprasad via
wangda)
Change-Id: I66aa4b0ec031ae5ce0fae558e2f8cbcbbfebc442
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/344c335a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/344c335a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/344c335a
Branch: refs/heads/trunk
Commit: 344c335a920e6f32a35ebace0a118a9dc4a22fb7
Parents: 5326a79
Author: Wangda Tan
Authored: Thu Aug 9 11:03:00 2018 -0700
Committer: Wangda Tan
Committed: Thu Aug 9 11:04:02 2018 -0700
--
.../capacity/AutoCreatedLeafQueueConfig.java| 5 ++
.../capacity/QueueManagementChange.java | 2 +-
.../QueueManagementDynamicEditPolicy.java | 36 ++
.../GuaranteedOrZeroCapacityOverTimePolicy.java | 50
4 files changed, 52 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/344c335a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
index 5952250..87ef1c0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
@@ -63,4 +63,9 @@ public class AutoCreatedLeafQueueConfig {
public CapacitySchedulerConfiguration getLeafQueueConfigs() {
return leafQueueConfigs;
}
+
+ @Override public String toString() {
+return "AutoCreatedLeafQueueConfig{" + "queueCapacities=" + queueCapacities
++ ", leafQueueConfigs=" + leafQueueConfigs + '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/344c335a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
index 74d9b23..64ba578 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
@@ -124,7 +124,7 @@ public abstract class QueueManagementChange {
@Override
public String toString() {
-return "QueueManagementChange{" + "queue=" + queue
+return "QueueManagementChange{" + "queue=" + queue.getQueueName()
+ ", updatedEntitlementsByPartition=" + queueTemplateUpdate
+ ", queueAction=" + queueAction + ", transitionToQueueState="
+ transitionToQueueState + '}';
http://git-wip-us.apache.org/repos/asf/hadoop/blob/344c335a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/cap
[1/2] hadoop git commit: YARN-8136. Add version attribute to site doc examples and quickstart. (Eric Yang via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 5326a7906 -> 8244abb7a
YARN-8136. Add version attribute to site doc examples and quickstart. (Eric
Yang via wangda)
Change-Id: I4541b239f490ca0a6edf9698e0d3deaf83669151
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8244abb7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8244abb7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8244abb7
Branch: refs/heads/trunk
Commit: 8244abb7aeb768b73682b8c9a26516a9cf06bca5
Parents: 344c335
Author: Wangda Tan
Authored: Thu Aug 9 11:03:46 2018 -0700
Committer: Wangda Tan
Committed: Thu Aug 9 11:04:02 2018 -0700
--
.../hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md | 5 -
.../src/site/markdown/yarn-service/QuickStart.md| 1 +
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8244abb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
index 03fec79..73e00b3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
@@ -30,6 +30,7 @@ Note this example requires registry DNS.
```
{
"name": "httpd-service",
+ "version": "1.0",
"lifetime": "3600",
"components": [
{
@@ -169,9 +170,10 @@ Then visit port 8080 for each IP to view the pages.
Docker images may have built with ENTRYPOINT to enable start up of docker
image without any parameters.
When passing parameters to ENTRYPOINT enabled image, `launch_command` is
delimited by comma (,).
+```
{
"name": "sleeper-service",
- "version": "1",
+ "version": "1.0",
"components" :
[
{
@@ -198,3 +200,4 @@ When passing parameters to ENTRYPOINT enabled image,
`launch_command` is delimit
}
]
}
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8244abb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index e144320..bba9bb8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -40,6 +40,7 @@ Below is a simple service definition that launches sleep
containers on YARN by w
```
{
"name": "sleeper-service",
+ "version": "1.0",
"components" :
[
{
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
