[2/2] hadoop git commit: YARN-3055. Fixed ResourceManager's DelegationTokenRenewer to not stop token renewal of applications part of a bigger workflow. Contributed by Daryn Sharp.

2015-09-01 Thread vinodkv 
YARN-3055. Fixed ResourceManager's DelegationTokenRenewer to not stop token 
renewal of applications part of a bigger workflow. Contributed by Daryn Sharp.

(cherry picked from commit 9c5911294e0ba71aefe4763731b0e780cde9d0ca)
(cherry picked from commit 1ff3fd33ed6f2ac09c774cc42b0107c5dbd9c19d)
(cherry picked from commit 82c722aae86669325672dd10840447434f15e7fd)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/752e3da7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/752e3da7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/752e3da7

Branch: refs/heads/branch-2.6.1
Commit: 752e3da738080cf7259b9085b736b42956d0bf1e
Parents: 619f793
Author: Vinod Kumar Vavilapalli 
Authored: Thu Apr 9 13:08:53 2015 -0700
Committer: Vinod Kumar Vavilapalli 
Committed: Tue Sep 1 21:31:00 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |   3 +
 .../security/DelegationTokenRenewer.java| 137 ---
 .../security/TestDelegationTokenRenewer.java|  87 +++-
 3 files changed, 173 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/752e3da7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d7e6622..fca01c0 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -120,6 +120,9 @@ Release 2.6.1 - UNRELEASED
 YARN-3393. Getting application(s) goes wrong when app finishes before
 starting the attempt. (Zhijie Shen via xgong)
 
+YARN-3055. Fixed ResourceManager's DelegationTokenRenewer to not stop token
+renewal of applications part of a bigger workflow. (Daryn Sharp via 
vinodkv)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/752e3da7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
index 70849fe..2cd31a1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
@@ -229,15 +230,16 @@ public class DelegationTokenRenewer extends 
AbstractService {
   @VisibleForTesting
   protected static class DelegationTokenToRenew {
 public final Token token;
-public final ApplicationId applicationId;
+public final Collection referringAppIds;
 public final Configuration conf;
 public long expirationDate;
-public TimerTask timerTask;
+public RenewalTimerTask timerTask;
 public volatile boolean shouldCancelAtEnd;
 public long maxDate;
 public String user;
 
-public DelegationTokenToRenew(ApplicationId jId, Token token,
+public DelegationTokenToRenew(Collection applicationIds,
+Token token,
 Configuration conf, long expirationDate, boolean shouldCancelAtEnd,
 String user) {
   this.token = token;
@@ -251,20 +253,33 @@ public class DelegationTokenRenewer extends 
AbstractService {
   throw new YarnRuntimeException(e);
 }
   }
-  this.applicationId = jId;
+  this.referringAppIds = Collections.synchronizedSet(
+  new HashSet(applicationIds));
   this.conf = conf;
   this.expirationDate = expirationDate;
   this.timerTask = null;
   this.shouldCancelAtEnd = shouldCancelAtEnd;
 }
 
-public void setTimerTask(TimerTask tTask) {
+public void setTimerTask(RenewalTimerTask tTask) {
   timerTask = tTask;
 }
-
+
+@VisibleForTesting
+public void cancelTimer() {
+  if (timerTask != null) {
+timerTask.cancel();
+  }
+}
+
+@VisibleForTesting
+

hadoop git commit: YARN-3055. Fixed ResourceManager's DelegationTokenRenewer to not stop token renewal of applications part of a bigger workflow. Contributed by Daryn Sharp.

2015-04-09 Thread vinodkv 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 4d5b1fbde - 4c6867ff0


YARN-3055. Fixed ResourceManager's DelegationTokenRenewer to not stop token 
renewal of applications part of a bigger workflow. Contributed by Daryn Sharp.

(cherry picked from commit 9c5911294e0ba71aefe4763731b0e780cde9d0ca)

Conflicts:
hadoop-yarn-project/CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c6867ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c6867ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c6867ff

Branch: refs/heads/branch-2.7
Commit: 4c6867ff01bf60ad7682184cd490924eb0ea81e5
Parents: 4d5b1fb
Author: Vinod Kumar Vavilapalli vino...@apache.org
Authored: Thu Apr 9 13:08:53 2015 -0700
Committer: Vinod Kumar Vavilapalli vino...@apache.org
Committed: Thu Apr 9 13:11:26 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |   3 +
 .../security/DelegationTokenRenewer.java| 137 ---
 .../security/TestDelegationTokenRenewer.java|  87 +++-
 3 files changed, 173 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c6867ff/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 28b33ff..43793f0 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -751,6 +751,9 @@ Release 2.7.0 - UNRELEASED
 YARN-3430. Made headroom data available on app attempt page of RM WebUI.
 (Xuan Gong via zjshen)
 
+YARN-3055. Fixed ResourceManager's DelegationTokenRenewer to not stop token
+renewal of applications part of a bigger workflow. (Daryn Sharp via 
vinodkv)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c6867ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
index 2619971..d49ecfc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
@@ -229,15 +230,16 @@ public class DelegationTokenRenewer extends 
AbstractService {
   @VisibleForTesting
   protected static class DelegationTokenToRenew {
 public final Token? token;
-public final ApplicationId applicationId;
+public final CollectionApplicationId referringAppIds;
 public final Configuration conf;
 public long expirationDate;
-public TimerTask timerTask;
+public RenewalTimerTask timerTask;
 public volatile boolean shouldCancelAtEnd;
 public long maxDate;
 public String user;
 
-public DelegationTokenToRenew(ApplicationId jId, Token? token,
+public DelegationTokenToRenew(CollectionApplicationId applicationIds,
+Token? token,
 Configuration conf, long expirationDate, boolean shouldCancelAtEnd,
 String user) {
   this.token = token;
@@ -251,20 +253,33 @@ public class DelegationTokenRenewer extends 
AbstractService {
   throw new YarnRuntimeException(e);
 }
   }
-  this.applicationId = jId;
+  this.referringAppIds = Collections.synchronizedSet(
+  new HashSetApplicationId(applicationIds));
   this.conf = conf;
   this.expirationDate = expirationDate;
   this.timerTask = null;
   this.shouldCancelAtEnd = shouldCancelAtEnd;
 }
 
-public void setTimerTask(TimerTask tTask) {
+public void setTimerTask(RenewalTimerTask tTask) {
   timerTask = tTask;
 }
-
+
+@VisibleForTesting
+public void cancelTimer() {
+  if (timerTask != null) {
+timerTask.cancel();
+  }
+}
+
+

hadoop git commit: YARN-3055. Fixed ResourceManager's DelegationTokenRenewer to not stop token renewal of applications part of a bigger workflow. Contributed by Daryn Sharp.

2015-04-09 Thread vinodkv 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 922b7ed21 - 9c5911294


YARN-3055. Fixed ResourceManager's DelegationTokenRenewer to not stop token 
renewal of applications part of a bigger workflow. Contributed by Daryn Sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c591129
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c591129
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c591129

Branch: refs/heads/trunk
Commit: 9c5911294e0ba71aefe4763731b0e780cde9d0ca
Parents: 922b7ed
Author: Vinod Kumar Vavilapalli vino...@apache.org
Authored: Thu Apr 9 13:08:53 2015 -0700
Committer: Vinod Kumar Vavilapalli vino...@apache.org
Committed: Thu Apr 9 13:08:53 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |   3 +
 .../security/DelegationTokenRenewer.java| 137 ---
 .../security/TestDelegationTokenRenewer.java|  87 +++-
 3 files changed, 173 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c591129/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c6c56d3..c29fdea 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -912,6 +912,9 @@ Release 2.7.0 - UNRELEASED
 YARN-3466. Fix RM nodes web page to sort by node HTTP-address, #containers 
 and node-label column (Jason Lowe via wangda)
 
+YARN-3055. Fixed ResourceManager's DelegationTokenRenewer to not stop token
+renewal of applications part of a bigger workflow. (Daryn Sharp via 
vinodkv)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c591129/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
index 2619971..d49ecfc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
@@ -229,15 +230,16 @@ public class DelegationTokenRenewer extends 
AbstractService {
   @VisibleForTesting
   protected static class DelegationTokenToRenew {
 public final Token? token;
-public final ApplicationId applicationId;
+public final CollectionApplicationId referringAppIds;
 public final Configuration conf;
 public long expirationDate;
-public TimerTask timerTask;
+public RenewalTimerTask timerTask;
 public volatile boolean shouldCancelAtEnd;
 public long maxDate;
 public String user;
 
-public DelegationTokenToRenew(ApplicationId jId, Token? token,
+public DelegationTokenToRenew(CollectionApplicationId applicationIds,
+Token? token,
 Configuration conf, long expirationDate, boolean shouldCancelAtEnd,
 String user) {
   this.token = token;
@@ -251,20 +253,33 @@ public class DelegationTokenRenewer extends 
AbstractService {
   throw new YarnRuntimeException(e);
 }
   }
-  this.applicationId = jId;
+  this.referringAppIds = Collections.synchronizedSet(
+  new HashSetApplicationId(applicationIds));
   this.conf = conf;
   this.expirationDate = expirationDate;
   this.timerTask = null;
   this.shouldCancelAtEnd = shouldCancelAtEnd;
 }
 
-public void setTimerTask(TimerTask tTask) {
+public void setTimerTask(RenewalTimerTask tTask) {
   timerTask = tTask;
 }
-
+
+@VisibleForTesting
+public void cancelTimer() {
+  if (timerTask != null) {
+timerTask.cancel();
+  }
+}
+
+@VisibleForTesting
+public boolean isTimerCancelled() {
+  return (timerTask != null)

hadoop git commit: YARN-3055. Fixed ResourceManager's DelegationTokenRenewer to not stop token renewal of applications part of a bigger workflow. Contributed by Daryn Sharp.

2015-04-09 Thread vinodkv 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 8dac24592 - 1ff3fd33e


YARN-3055. Fixed ResourceManager's DelegationTokenRenewer to not stop token 
renewal of applications part of a bigger workflow. Contributed by Daryn Sharp.

(cherry picked from commit 9c5911294e0ba71aefe4763731b0e780cde9d0ca)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ff3fd33
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ff3fd33
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ff3fd33

Branch: refs/heads/branch-2
Commit: 1ff3fd33ed6f2ac09c774cc42b0107c5dbd9c19d
Parents: 8dac245
Author: Vinod Kumar Vavilapalli vino...@apache.org
Authored: Thu Apr 9 13:08:53 2015 -0700
Committer: Vinod Kumar Vavilapalli vino...@apache.org
Committed: Thu Apr 9 13:10:06 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |   3 +
 .../security/DelegationTokenRenewer.java| 137 ---
 .../security/TestDelegationTokenRenewer.java|  87 +++-
 3 files changed, 173 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff3fd33/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6a3999f..048557d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -867,6 +867,9 @@ Release 2.7.0 - UNRELEASED
 YARN-3466. Fix RM nodes web page to sort by node HTTP-address, #containers 
 and node-label column (Jason Lowe via wangda)
 
+YARN-3055. Fixed ResourceManager's DelegationTokenRenewer to not stop token
+renewal of applications part of a bigger workflow. (Daryn Sharp via 
vinodkv)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ff3fd33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
index 2619971..d49ecfc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
@@ -229,15 +230,16 @@ public class DelegationTokenRenewer extends 
AbstractService {
   @VisibleForTesting
   protected static class DelegationTokenToRenew {
 public final Token? token;
-public final ApplicationId applicationId;
+public final CollectionApplicationId referringAppIds;
 public final Configuration conf;
 public long expirationDate;
-public TimerTask timerTask;
+public RenewalTimerTask timerTask;
 public volatile boolean shouldCancelAtEnd;
 public long maxDate;
 public String user;
 
-public DelegationTokenToRenew(ApplicationId jId, Token? token,
+public DelegationTokenToRenew(CollectionApplicationId applicationIds,
+Token? token,
 Configuration conf, long expirationDate, boolean shouldCancelAtEnd,
 String user) {
   this.token = token;
@@ -251,20 +253,33 @@ public class DelegationTokenRenewer extends 
AbstractService {
   throw new YarnRuntimeException(e);
 }
   }
-  this.applicationId = jId;
+  this.referringAppIds = Collections.synchronizedSet(
+  new HashSetApplicationId(applicationIds));
   this.conf = conf;
   this.expirationDate = expirationDate;
   this.timerTask = null;
   this.shouldCancelAtEnd = shouldCancelAtEnd;
 }
 
-public void setTimerTask(TimerTask tTask) {
+public void setTimerTask(RenewalTimerTask tTask) {
   timerTask = tTask;
 }
-
+
+@VisibleForTesting
+public void cancelTimer() {
+  if (timerTask != null) {
+timerTask.cancel();
+  }
+}
+
+@VisibleForTesting
+public boolean