[18/50] [abbrv] hadoop git commit: YARN-8869. YARN Service Client might not work correctly with RM REST API for Kerberos authentication. Contributed by Eric Yang.

2018-10-16 Thread shv 
YARN-8869. YARN Service Client might not work correctly with RM REST API for 
Kerberos authentication. Contributed by Eric Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa94d370
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa94d370
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa94d370

Branch: refs/heads/HDFS-12943
Commit: fa94d370b6e3cba9c7560c09b517583e6652f103
Parents: b4a38e7
Author: Sunil G 
Authored: Mon Oct 15 21:21:57 2018 +0530
Committer: Sunil G 
Committed: Mon Oct 15 21:21:57 2018 +0530

--
 .../yarn/service/client/ApiServiceClient.java   |  19 ++--
 .../client/TestSecureApiServiceClient.java  | 113 ++-
 2 files changed, 120 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa94d370/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index b7a1541..851acbd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -70,7 +70,6 @@ import org.slf4j.LoggerFactory;
 
 import com.sun.jersey.api.client.Client;
 import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.api.client.WebResource.Builder;
 import com.sun.jersey.api.client.config.ClientConfig;
 import com.sun.jersey.api.client.config.DefaultClientConfig;
@@ -144,7 +143,7 @@ public class ApiServiceClient extends AppAdminClient {
   /**
* Calculate Resource Manager address base on working REST API.
*/
-  private String getRMWebAddress() {
+  String getRMWebAddress() {
 Configuration conf = getConfig();
 String scheme = "http://;;
 String path = "/app/v1/services/version";
@@ -156,8 +155,7 @@ public class ApiServiceClient extends AppAdminClient {
   .get("yarn.resourcemanager.webapp.https.address");
 }
 boolean useKerberos = UserGroupInformation.isSecurityEnabled();
-List rmServers = RMHAUtils
-.getRMHAWebappAddresses(new YarnConfiguration(conf));
+List rmServers = getRMHAWebAddresses(conf);
 for (String host : rmServers) {
   try {
 Client client = Client.create();
@@ -175,16 +173,16 @@ public class ApiServiceClient extends AppAdminClient {
 LOG.debug("Fail to resolve username: {}", e);
   }
 }
-WebResource webResource = client
-.resource(sb.toString());
+Builder builder = client
+.resource(sb.toString()).type(MediaType.APPLICATION_JSON);
 if (useKerberos) {
   String[] server = host.split(":");
   String challenge = generateToken(server[0]);
-  webResource.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
+  builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
   challenge);
   LOG.debug("Authorization: Negotiate {}", challenge);
 }
-ClientResponse test = webResource.get(ClientResponse.class);
+ClientResponse test = builder.get(ClientResponse.class);
 if (test.getStatus() == 200) {
   rmAddress = host;
   break;
@@ -197,6 +195,11 @@ public class ApiServiceClient extends AppAdminClient {
 return scheme+rmAddress;
   }
 
+  List getRMHAWebAddresses(Configuration conf) {
+return RMHAUtils
+.getRMHAWebappAddresses(new YarnConfiguration(conf));
+  }
+
   /**
* Compute active resource manager API service location.
*

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa94d370/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java

hadoop git commit: YARN-8869. YARN Service Client might not work correctly with RM REST API for Kerberos authentication. Contributed by Eric Yang.

2018-10-15 Thread sunilg 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 6a1ce74fb -> 38e187384


YARN-8869. YARN Service Client might not work correctly with RM REST API for 
Kerberos authentication. Contributed by Eric Yang.

(cherry picked from commit fa94d370b6e3cba9c7560c09b517583e6652f103)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38e18738
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38e18738
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38e18738

Branch: refs/heads/branch-3.1
Commit: 38e187384b2cd1894f0dcf97c337b2de6fd86986
Parents: 6a1ce74
Author: Sunil G 
Authored: Mon Oct 15 21:21:57 2018 +0530
Committer: Sunil G 
Committed: Mon Oct 15 21:23:07 2018 +0530

--
 .../yarn/service/client/ApiServiceClient.java   |  19 ++--
 .../client/TestSecureApiServiceClient.java  | 113 ++-
 2 files changed, 120 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/38e18738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index e6dee79..074b45d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -70,7 +70,6 @@ import org.slf4j.LoggerFactory;
 
 import com.sun.jersey.api.client.Client;
 import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.api.client.WebResource.Builder;
 import com.sun.jersey.api.client.config.ClientConfig;
 import com.sun.jersey.api.client.config.DefaultClientConfig;
@@ -144,7 +143,7 @@ public class ApiServiceClient extends AppAdminClient {
   /**
* Calculate Resource Manager address base on working REST API.
*/
-  private String getRMWebAddress() {
+  String getRMWebAddress() {
 Configuration conf = getConfig();
 String scheme = "http://;;
 String path = "/app/v1/services/version";
@@ -156,8 +155,7 @@ public class ApiServiceClient extends AppAdminClient {
   .get("yarn.resourcemanager.webapp.https.address");
 }
 boolean useKerberos = UserGroupInformation.isSecurityEnabled();
-List rmServers = RMHAUtils
-.getRMHAWebappAddresses(new YarnConfiguration(conf));
+List rmServers = getRMHAWebAddresses(conf);
 for (String host : rmServers) {
   try {
 Client client = Client.create();
@@ -175,16 +173,16 @@ public class ApiServiceClient extends AppAdminClient {
 LOG.debug("Fail to resolve username: {}", e);
   }
 }
-WebResource webResource = client
-.resource(sb.toString());
+Builder builder = client
+.resource(sb.toString()).type(MediaType.APPLICATION_JSON);
 if (useKerberos) {
   String[] server = host.split(":");
   String challenge = generateToken(server[0]);
-  webResource.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
+  builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
   challenge);
   LOG.debug("Authorization: Negotiate {}", challenge);
 }
-ClientResponse test = webResource.get(ClientResponse.class);
+ClientResponse test = builder.get(ClientResponse.class);
 if (test.getStatus() == 200) {
   rmAddress = host;
   break;
@@ -197,6 +195,11 @@ public class ApiServiceClient extends AppAdminClient {
 return scheme+rmAddress;
   }
 
+  List getRMHAWebAddresses(Configuration conf) {
+return RMHAUtils
+.getRMHAWebappAddresses(new YarnConfiguration(conf));
+  }
+
   /**
* Compute active resource manager API service location.
*

http://git-wip-us.apache.org/repos/asf/hadoop/blob/38e18738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
--
diff --git

hadoop git commit: YARN-8869. YARN Service Client might not work correctly with RM REST API for Kerberos authentication. Contributed by Eric Yang.

2018-10-15 Thread sunilg 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.2 aff21d727 -> 1ca5f974f


YARN-8869. YARN Service Client might not work correctly with RM REST API for 
Kerberos authentication. Contributed by Eric Yang.

(cherry picked from commit fa94d370b6e3cba9c7560c09b517583e6652f103)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ca5f974
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ca5f974
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ca5f974

Branch: refs/heads/branch-3.2
Commit: 1ca5f974f13baeed375b62b233c7845933a87f11
Parents: aff21d7
Author: Sunil G 
Authored: Mon Oct 15 21:21:57 2018 +0530
Committer: Sunil G 
Committed: Mon Oct 15 21:22:45 2018 +0530

--
 .../yarn/service/client/ApiServiceClient.java   |  19 ++--
 .../client/TestSecureApiServiceClient.java  | 113 ++-
 2 files changed, 120 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ca5f974/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index b7a1541..851acbd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -70,7 +70,6 @@ import org.slf4j.LoggerFactory;
 
 import com.sun.jersey.api.client.Client;
 import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.api.client.WebResource.Builder;
 import com.sun.jersey.api.client.config.ClientConfig;
 import com.sun.jersey.api.client.config.DefaultClientConfig;
@@ -144,7 +143,7 @@ public class ApiServiceClient extends AppAdminClient {
   /**
* Calculate Resource Manager address base on working REST API.
*/
-  private String getRMWebAddress() {
+  String getRMWebAddress() {
 Configuration conf = getConfig();
 String scheme = "http://;;
 String path = "/app/v1/services/version";
@@ -156,8 +155,7 @@ public class ApiServiceClient extends AppAdminClient {
   .get("yarn.resourcemanager.webapp.https.address");
 }
 boolean useKerberos = UserGroupInformation.isSecurityEnabled();
-List rmServers = RMHAUtils
-.getRMHAWebappAddresses(new YarnConfiguration(conf));
+List rmServers = getRMHAWebAddresses(conf);
 for (String host : rmServers) {
   try {
 Client client = Client.create();
@@ -175,16 +173,16 @@ public class ApiServiceClient extends AppAdminClient {
 LOG.debug("Fail to resolve username: {}", e);
   }
 }
-WebResource webResource = client
-.resource(sb.toString());
+Builder builder = client
+.resource(sb.toString()).type(MediaType.APPLICATION_JSON);
 if (useKerberos) {
   String[] server = host.split(":");
   String challenge = generateToken(server[0]);
-  webResource.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
+  builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
   challenge);
   LOG.debug("Authorization: Negotiate {}", challenge);
 }
-ClientResponse test = webResource.get(ClientResponse.class);
+ClientResponse test = builder.get(ClientResponse.class);
 if (test.getStatus() == 200) {
   rmAddress = host;
   break;
@@ -197,6 +195,11 @@ public class ApiServiceClient extends AppAdminClient {
 return scheme+rmAddress;
   }
 
+  List getRMHAWebAddresses(Configuration conf) {
+return RMHAUtils
+.getRMHAWebappAddresses(new YarnConfiguration(conf));
+  }
+
   /**
* Compute active resource manager API service location.
*

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ca5f974/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
--
diff --git

hadoop git commit: YARN-8869. YARN Service Client might not work correctly with RM REST API for Kerberos authentication. Contributed by Eric Yang.

2018-10-15 Thread sunilg 
Repository: hadoop
Updated Branches:
  refs/heads/trunk b4a38e7b3 -> fa94d370b


YARN-8869. YARN Service Client might not work correctly with RM REST API for 
Kerberos authentication. Contributed by Eric Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa94d370
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa94d370
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa94d370

Branch: refs/heads/trunk
Commit: fa94d370b6e3cba9c7560c09b517583e6652f103
Parents: b4a38e7
Author: Sunil G 
Authored: Mon Oct 15 21:21:57 2018 +0530
Committer: Sunil G 
Committed: Mon Oct 15 21:21:57 2018 +0530

--
 .../yarn/service/client/ApiServiceClient.java   |  19 ++--
 .../client/TestSecureApiServiceClient.java  | 113 ++-
 2 files changed, 120 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa94d370/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index b7a1541..851acbd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -70,7 +70,6 @@ import org.slf4j.LoggerFactory;
 
 import com.sun.jersey.api.client.Client;
 import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.api.client.WebResource.Builder;
 import com.sun.jersey.api.client.config.ClientConfig;
 import com.sun.jersey.api.client.config.DefaultClientConfig;
@@ -144,7 +143,7 @@ public class ApiServiceClient extends AppAdminClient {
   /**
* Calculate Resource Manager address base on working REST API.
*/
-  private String getRMWebAddress() {
+  String getRMWebAddress() {
 Configuration conf = getConfig();
 String scheme = "http://;;
 String path = "/app/v1/services/version";
@@ -156,8 +155,7 @@ public class ApiServiceClient extends AppAdminClient {
   .get("yarn.resourcemanager.webapp.https.address");
 }
 boolean useKerberos = UserGroupInformation.isSecurityEnabled();
-List rmServers = RMHAUtils
-.getRMHAWebappAddresses(new YarnConfiguration(conf));
+List rmServers = getRMHAWebAddresses(conf);
 for (String host : rmServers) {
   try {
 Client client = Client.create();
@@ -175,16 +173,16 @@ public class ApiServiceClient extends AppAdminClient {
 LOG.debug("Fail to resolve username: {}", e);
   }
 }
-WebResource webResource = client
-.resource(sb.toString());
+Builder builder = client
+.resource(sb.toString()).type(MediaType.APPLICATION_JSON);
 if (useKerberos) {
   String[] server = host.split(":");
   String challenge = generateToken(server[0]);
-  webResource.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
+  builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
   challenge);
   LOG.debug("Authorization: Negotiate {}", challenge);
 }
-ClientResponse test = webResource.get(ClientResponse.class);
+ClientResponse test = builder.get(ClientResponse.class);
 if (test.getStatus() == 200) {
   rmAddress = host;
   break;
@@ -197,6 +195,11 @@ public class ApiServiceClient extends AppAdminClient {
 return scheme+rmAddress;
   }
 
+  List getRMHAWebAddresses(Configuration conf) {
+return RMHAUtils
+.getRMHAWebappAddresses(new YarnConfiguration(conf));
+  }
+
   /**
* Compute active resource manager API service location.
*

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa94d370/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
--
diff --git