Repository: zeppelin Updated Branches: refs/heads/master 13f6dc7de -> 02daea1c5
[ZEPPELIN-2965] Add code completion for livy interpreter ### What is this PR for? This PR adds code autocompletion feature to LivyInterpreter. Livy version 0.5 will have an auto completion API. ### What type of PR is it? Feature ### Todos * [ ] - Task ### What is the Jira issue? [ZEPPELIN-2965] https://issues.apache.org/jira/browse/ZEPPELIN-2965 ### How should this be tested? Pulled out server calls to a separate class to support proper unit-testing with mockito. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Pascal Pellmont <[email protected]> Closes #2624 from pellmont/ZEPPELIN-2965 and squashes the following commits: aeabf86 [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - added missing commas in interpreter-settings.json 55515a7 [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - fixed checkstyle violation 103681e [Pascal Pellmont] Merge branch 'master' of github.com:apache/zeppelin into ZEPPELIN-2965 d35a90f [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - added missing commas in interpreter-settings.json bbb48fb [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - removed pointless try 2a21f3c [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - no new session for code completion a460ae7 [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - fixed typo 9fac364 [Pascal Pellmont] ZEPPELIN-2965 code completion for livy - TAB as completion key 32b3c6b [Pascal Pellmont] ZEPPELIN-2965 code completion for livy Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/02daea1c Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/02daea1c Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/02daea1c Branch: refs/heads/master Commit: 02daea1c596939592d0d7899600c4243759d5fbf Parents: 13f6dc7 Author: Pascal Pellmont <[email protected]> Authored: Thu Nov 2 07:12:07 2017 +0100 Committer: Jeff Zhang <[email protected]> Committed: Thu Nov 2 15:47:39 2017 +0800 ---------------------------------------------------------------------- .../zeppelin/livy/BaseLivyInterpreter.java | 107 +++++++++++++++---- .../src/main/resources/interpreter-setting.json | 15 ++- 2 files changed, 97 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/02daea1c/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java ---------------------------------------------------------------------- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index 2122f53..0cdf464 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -17,10 +17,25 @@ package org.apache.zeppelin.livy; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.annotations.SerializedName; +import java.io.FileInputStream; +import java.io.IOException; +import java.security.KeyStore; +import java.security.Principal; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.net.ssl.SSLContext; + import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.http.auth.AuthSchemeProvider; import org.apache.http.auth.AuthScope; import org.apache.http.auth.Credentials; @@ -36,14 +51,19 @@ import org.apache.http.impl.auth.SPNegoSchemeFactory; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; -import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.InterpreterUtils; +import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpEntity; -import org.springframework.http.MediaType; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; import org.springframework.security.kerberos.client.KerberosRestTemplate; @@ -51,20 +71,10 @@ import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.HttpServerErrorException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; -import javax.net.ssl.SSLContext; -import java.io.FileInputStream; -import java.io.IOException; -import java.security.KeyStore; -import java.security.Principal; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.regex.Matcher; -import java.util.regex.Pattern; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.annotations.SerializedName; /** @@ -205,6 +215,35 @@ public abstract class BaseLivyInterpreter extends Interpreter { } @Override + public List<InterpreterCompletion> completion(String buf, int cursor, + InterpreterContext interpreterContext) { + List<InterpreterCompletion> candidates = Collections.emptyList(); + try { + candidates = callCompletion(new CompletionRequest(buf, getSessionKind(), cursor)); + } catch (SessionNotFoundException e) { + LOGGER.warn("Livy session {} is expired. Will return empty list of candidates.", + sessionInfo.id); + } catch (LivyException le) { + logger.error("Failed to call code completions. Will return empty list of candidates", le); + } + return candidates; + } + + private List<InterpreterCompletion> callCompletion(CompletionRequest req) throws LivyException { + List<InterpreterCompletion> candidates = new ArrayList<>(); + try { + CompletionResponse resp = CompletionResponse.fromJson( + callRestAPI("/sessions/" + sessionInfo.id + "/completion", "POST", req.toJson())); + for (String candidate : resp.candidates) { + candidates.add(new InterpreterCompletion(candidate, candidate, StringUtils.EMPTY)); + } + } catch (APINotFoundException e) { + logger.debug("completion api seems not to be available. (available from livy 0.5)", e); + } + return candidates; + } + + @Override public void cancel(InterpreterContext context) { paragraphsToCancel.add(context.getParagraphId()); LOGGER.info("Added paragraph " + context.getParagraphId() + " for cancellation."); @@ -774,6 +813,34 @@ public abstract class BaseLivyInterpreter extends Interpreter { } } + static class CompletionRequest { + public final String code; + public final String kind; + public final int cursor; + + public CompletionRequest(String code, String kind, int cursor) { + this.code = code; + this.kind = kind; + this.cursor = cursor; + } + + public String toJson() { + return gson.toJson(this); + } + } + + static class CompletionResponse { + public final String[] candidates; + + public CompletionResponse(String[] candidates) { + this.candidates = candidates; + } + + public static CompletionResponse fromJson(String json) { + return gson.fromJson(json, CompletionResponse.class); + } + } + private static class LivyVersionResponse { public String url; public String branch; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/02daea1c/livy/src/main/resources/interpreter-setting.json ---------------------------------------------------------------------- diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json index ac213ba..2d72487 100644 --- a/livy/src/main/resources/interpreter-setting.json +++ b/livy/src/main/resources/interpreter-setting.json @@ -121,7 +121,8 @@ }, "editor": { "language": "scala", - "editOnDblClick": false + "editOnDblClick": false, + "completionKey": "TAB" } }, { @@ -160,7 +161,8 @@ }, "editor": { "language": "sql", - "editOnDblClick": false + "editOnDblClick": false, + "completionKey": "TAB" } }, { @@ -180,7 +182,8 @@ }, "editor": { "language": "python", - "editOnDblClick": false + "editOnDblClick": false, + "completionKey": "TAB" } }, { @@ -200,7 +203,8 @@ }, "editor": { "language": "python", - "editOnDblClick": false + "editOnDblClick": false, + "completionKey": "TAB" } }, { @@ -220,7 +224,8 @@ }, "editor": { "language": "r", - "editOnDblClick": false + "editOnDblClick": false, + "completionKey": "TAB" } } ]
