Repository: zeppelin Updated Branches: refs/heads/master c06c375e9 -> 4044189de
ZEPPELIN-1037 Enable Kerberos support in Livy Interpreter ### What is this PR for? This PR is for enabling kerberos support in Livy interpreter. Also introduced two new Livy interpreter properties called keytab and principal to configure. ### What type of PR is it? Improvement ### Todos ### What is the Jira issue? ZEPPELIN-1037 ### How should this be tested? Kerberize the Livy server and configure the Livy interpreter with appropriate keytab and principal. ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? yes Author: Renjith Kamath <[email protected]> Closes #1052 from r-kamath/ZEPPELIN-1037 and squashes the following commits: 3bf7269 [Renjith Kamath] ZEPPELIN-1037 Enable Kerberos support in livy 1cc8c4d [Renjith Kamath] ZEPPELIN-1037 Enable Kerberos support in Livy Interpreter Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/4044189d Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/4044189d Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/4044189d Branch: refs/heads/master Commit: 4044189dec42fed80f0d2b09890df8961d4a57f4 Parents: c06c375 Author: Renjith Kamath <[email protected]> Authored: Tue Jun 21 23:35:14 2016 +0530 Committer: Prabhjyot Singh <[email protected]> Committed: Mon Jun 27 21:16:35 2016 +0530 ---------------------------------------------------------------------- livy/pom.xml | 11 +++ .../org/apache/zeppelin/livy/LivyHelper.java | 92 +++++++++----------- .../src/main/resources/interpreter-setting.json | 10 +++ 3 files changed, 60 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4044189d/livy/pom.xml ---------------------------------------------------------------------- diff --git a/livy/pom.xml b/livy/pom.xml index 90f149c..6fa12b9 100644 --- a/livy/pom.xml +++ b/livy/pom.xml @@ -96,6 +96,17 @@ <version>${mockito.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.springframework.security.kerberos</groupId> + <artifactId>spring-security-kerberos-client</artifactId> + <version>1.0.1.RELEASE</version> + </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-web</artifactId> + <version>4.3.0.RELEASE</version> + </dependency> + </dependencies> <build> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4044189d/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java ---------------------------------------------------------------------- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java index 8c4ddab..2c66fa9 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java @@ -21,22 +21,20 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.HttpClientBuilder; + import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.security.kerberos.client.KerberosRestTemplate; +import org.springframework.web.client.RestTemplate; -import java.io.BufferedReader; -import java.io.InputStreamReader; import java.nio.charset.Charset; import java.util.*; import java.util.Map.Entry; @@ -69,14 +67,14 @@ public class LivyHelper { } String confData = gson.toJson(conf); + String user = context.getAuthenticationInfo().getUser(); - String json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions", - "POST", + String json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions", "POST", "{" + "\"kind\": \"" + kind + "\", " + "\"conf\": " + confData + ", " + - "\"proxyUser\": \"" + context.getAuthenticationInfo().getUser() + - "\"}", + "\"proxyUser\": " + (StringUtils.isEmpty(user) ? null : "\"" + user + "\"") + + "}", context.getParagraphId() ); @@ -329,66 +327,54 @@ public class LivyHelper { } } + private RestTemplate getRestTemplate() { + String keytabLocation = property.getProperty("zeppelin.livy.keytab"); + String principal = property.getProperty("zeppelin.livy.principal"); + if (StringUtils.isNotEmpty(keytabLocation) && StringUtils.isNotEmpty(principal)) { + return new KerberosRestTemplate(keytabLocation, principal); + } + return new RestTemplate(); + } + protected String executeHTTP(String targetURL, String method, String jsonData, String paragraphId) throws Exception { - HttpClient client = HttpClientBuilder.create().build(); - HttpResponse response = null; + RestTemplate restTemplate = getRestTemplate(); + HttpHeaders headers = new HttpHeaders(); + headers.add("Content-Type", "application/json"); + ResponseEntity<String> response = null; if (method.equals("POST")) { - HttpPost request = new HttpPost(targetURL); - request.addHeader("Content-Type", "application/json"); - StringEntity se = new StringEntity(jsonData); - request.setEntity(se); - response = client.execute(request); - paragraphHttpMap.put(paragraphId, request); + HttpEntity<String> entity = new HttpEntity<String>(jsonData, headers); + response = restTemplate.exchange(targetURL, HttpMethod.POST, entity, String.class); + paragraphHttpMap.put(paragraphId, response); } else if (method.equals("GET")) { - HttpGet request = new HttpGet(targetURL); - request.addHeader("Content-Type", "application/json"); - response = client.execute(request); - paragraphHttpMap.put(paragraphId, request); + HttpEntity<String> entity = new HttpEntity<String>(headers); + response = restTemplate.exchange(targetURL, HttpMethod.GET, entity, String.class); + paragraphHttpMap.put(paragraphId, response); } else if (method.equals("DELETE")) { - HttpDelete request = new HttpDelete(targetURL); - request.addHeader("Content-Type", "application/json"); - response = client.execute(request); + HttpEntity<String> entity = new HttpEntity<String>(headers); + response = restTemplate.exchange(targetURL, HttpMethod.DELETE, entity, String.class); } - if (response == null) { return null; } - if (response.getStatusLine().getStatusCode() == 200 - || response.getStatusLine().getStatusCode() == 201 - || response.getStatusLine().getStatusCode() == 404) { - return getResponse(response); + if (response.getStatusCode().value() == 200 + || response.getStatusCode().value() == 201 + || response.getStatusCode().value() == 404) { + return response.getBody(); } else { - String responseString = getResponse(response); + String responseString = response.getBody(); if (responseString.contains("CreateInteractiveRequest[\\\"master\\\"]")) { return responseString; } LOGGER.error(String.format("Error with %s StatusCode: %s", - response.getStatusLine().getStatusCode(), responseString)); + response.getStatusCode().value(), responseString)); throw new Exception(String.format("Error with %s StatusCode: %s", - response.getStatusLine().getStatusCode(), responseString)); + response.getStatusCode().value(), responseString)); } } - private String getResponse(HttpResponse response) throws Exception { - BufferedReader rd = new BufferedReader( - new InputStreamReader(response.getEntity().getContent())); - - StringBuffer result = new StringBuffer(); - String line = ""; - while ((line = rd.readLine()) != null) { - result.append(line); - } - return result.toString(); - } - public void cancelHTTP(String paragraphId) { - if (paragraphHttpMap.get(paragraphId).getClass().getName().contains("HttpPost")) { - ((HttpPost) paragraphHttpMap.get(paragraphId)).abort(); - } else { - ((HttpGet) paragraphHttpMap.get(paragraphId)).abort(); - } paragraphHttpMap.put(paragraphId, null); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4044189d/livy/src/main/resources/interpreter-setting.json ---------------------------------------------------------------------- diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json index c22e9a7..7ae435f 100644 --- a/livy/src/main/resources/interpreter-setting.json +++ b/livy/src/main/resources/interpreter-setting.json @@ -64,6 +64,16 @@ "propertyName": "livy.spark.dynamicAllocation.maxExecutors", "defaultValue": "", "description": "Upper bound for the number of executors if dynamic allocation is enabled." + }, + "zeppelin.livy.principal": { + "propertyName": "zeppelin.livy.principal", + "defaultValue": "", + "description": "Kerberos principal to authenticate livy" + }, + "zeppelin.livy.keytab": { + "propertyName": "zeppelin.livy.keytab", + "defaultValue": "", + "description": "Kerberos keytab to authenticate livy" } } },
