http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java index 5eb34e4..43639f7 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java @@ -16,18 +16,6 @@ */ package org.apache.zeppelin.python; -import org.apache.commons.lang.StringUtils; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterOutput; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.InterpreterResult.Type; -import org.apache.zeppelin.scheduler.Scheduler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -40,11 +28,19 @@ import java.util.Map; import java.util.Properties; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.InterpreterResult.Type; +import org.apache.zeppelin.scheduler.Scheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Conda support - * TODO(zjffdu) Add removing conda env - */ +/** Conda support TODO(zjffdu) Add removing conda env */ public class PythonCondaInterpreter extends Interpreter { private static Logger logger = LoggerFactory.getLogger(PythonCondaInterpreter.class); public static final String ZEPPELIN_PYTHON = "zeppelin.python"; @@ -70,14 +66,10 @@ public class PythonCondaInterpreter extends Interpreter { } @Override - public void open() throws InterpreterException { - - } + public void open() throws InterpreterException {} @Override - public void close() { - - } + public void close() {} @Override public InterpreterResult interpret(String st, InterpreterContext context) @@ -167,7 +159,6 @@ public class PythonCondaInterpreter extends Interpreter { getInterpreterInTheSameSessionByClassName(PythonInterpreter.class, false); pythonInterpreter.close(); pythonInterpreter.open(); - } public static String runCondaCommandForTextOutput(String title, List<String> commands) @@ -188,8 +179,7 @@ public class PythonCondaInterpreter extends Interpreter { return wrapCondaTableOutputStyle(title, envPerName); } - protected Map<String, String> getCondaEnvs() - throws IOException, InterruptedException { + protected Map<String, String> getCondaEnvs() throws IOException, InterruptedException { String result = runCommand("conda", "env", "list"); Map<String, String> envList = parseCondaCommonStdout(result); return envList; @@ -199,8 +189,7 @@ public class PythonCondaInterpreter extends Interpreter { return wrapCondaTableOutputStyle("Environment List", getCondaEnvs()); } - private String runCondaEnv(List<String> restArgs) - throws IOException, InterruptedException { + private String runCondaEnv(List<String> restArgs) throws IOException, InterruptedException { restArgs.add(0, "conda"); restArgs.add(1, "env"); @@ -259,8 +248,7 @@ public class PythonCondaInterpreter extends Interpreter { return runCondaCommandForTextOutput("Conda Information", commands); } - private String runCondaCreate(List<String> restArgs) - throws IOException, InterruptedException { + private String runCondaCreate(List<String> restArgs) throws IOException, InterruptedException { restArgs.add(0, "conda"); restArgs.add(1, "create"); restArgs.add(2, "--yes"); @@ -268,8 +256,7 @@ public class PythonCondaInterpreter extends Interpreter { return runCondaCommandForTextOutput("Environment Creation", restArgs); } - private String runCondaInstall(List<String> restArgs) - throws IOException, InterruptedException { + private String runCondaInstall(List<String> restArgs) throws IOException, InterruptedException { restArgs.add(0, "conda"); restArgs.add(1, "install"); @@ -282,8 +269,7 @@ public class PythonCondaInterpreter extends Interpreter { return runCondaCommandForTextOutput("Package Installation", restArgs); } - private String runCondaUninstall(List<String> restArgs) - throws IOException, InterruptedException { + private String runCondaUninstall(List<String> restArgs) throws IOException, InterruptedException { restArgs.add(0, "conda"); restArgs.add(1, "uninstall"); @@ -299,12 +285,9 @@ public class PythonCondaInterpreter extends Interpreter { public static String wrapCondaBasicOutputStyle(String title, String content) { StringBuilder sb = new StringBuilder(); if (null != title && !title.isEmpty()) { - sb.append("<h4>").append(title).append("</h4>\n") - .append("</div><br />\n"); + sb.append("<h4>").append(title).append("</h4>\n").append("</div><br />\n"); } - sb.append("<div style=\"white-space:pre-wrap;\">\n") - .append(content) - .append("</div>"); + sb.append("<div style=\"white-space:pre-wrap;\">\n").append(content).append("</div>"); return sb.toString(); } @@ -320,11 +303,13 @@ public class PythonCondaInterpreter extends Interpreter { for (String name : kv.keySet()) { String path = kv.get(name); - sb.append(String.format("<div style=\"display:table-row\">" + - "<div style=\"display:table-cell;width:150px\">%s</div>" + - "<div style=\"display:table-cell;\">%s</div>" + - "</div>\n", - name, path)); + sb.append( + String.format( + "<div style=\"display:table-row\">" + + "<div style=\"display:table-cell;width:150px\">%s</div>" + + "<div style=\"display:table-cell;\">%s</div>" + + "</div>\n", + name, path)); } sb.append("</div>\n"); @@ -352,9 +337,7 @@ public class PythonCondaInterpreter extends Interpreter { } @Override - public void cancel(InterpreterContext context) { - - } + public void cancel(InterpreterContext context) {} @Override public FormType getFormType() { @@ -367,8 +350,8 @@ public class PythonCondaInterpreter extends Interpreter { } /** - * Use python interpreter's scheduler. - * To make sure %python.conda paragraph and %python paragraph runs sequentially + * Use python interpreter's scheduler. To make sure %python.conda paragraph and %python paragraph + * runs sequentially */ @Override public Scheduler getScheduler() { @@ -381,8 +364,7 @@ public class PythonCondaInterpreter extends Interpreter { } } - public static String runCommand(List<String> commands) - throws IOException, InterruptedException { + public static String runCommand(List<String> commands) throws IOException, InterruptedException { logger.info("Starting shell commands: " + StringUtils.join(commands, " ")); Process process = Runtime.getRuntime().exec(commands.toArray(new String[0])); StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream()); @@ -429,8 +411,7 @@ public class PythonCondaInterpreter extends Interpreter { } } - public static String runCommand(String... command) - throws IOException, InterruptedException { + public static String runCommand(String... command) throws IOException, InterruptedException { List<String> list = new ArrayList<>(command.length); for (String arg : command) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java index 52caf35..1774187 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java @@ -16,15 +16,6 @@ */ package org.apache.zeppelin.python; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterOutput; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.scheduler.Scheduler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -34,10 +25,16 @@ import java.nio.file.Paths; import java.util.Properties; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.scheduler.Scheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Helps run python interpreter on a docker container - */ +/** Helps run python interpreter on a docker container */ public class PythonDockerInterpreter extends Interpreter { Logger logger = LoggerFactory.getLogger(PythonDockerInterpreter.class); Pattern activatePattern = Pattern.compile("activate\\s*(.*)"); @@ -61,9 +58,7 @@ public class PythonDockerInterpreter extends Interpreter { } @Override - public void close() { - - } + public void close() {} @Override public InterpreterResult interpret(String st, InterpreterContext context) @@ -83,23 +78,26 @@ public class PythonDockerInterpreter extends Interpreter { pull(out, image); // mount pythonscript dir - String mountPythonScript = "-v " + pythonWorkDir.getAbsolutePath() + - ":/_python_workdir "; + String mountPythonScript = "-v " + pythonWorkDir.getAbsolutePath() + ":/_python_workdir "; // mount zeppelin dir - String mountPy4j = "-v " + zeppelinHome.getAbsolutePath() + - ":/_zeppelin "; + String mountPy4j = "-v " + zeppelinHome.getAbsolutePath() + ":/_zeppelin "; // set PYTHONPATH String pythonPath = ".:/_python_workdir/py4j-src-0.10.7.zip:/_python_workdir"; - setPythonCommand("docker run -i --rm " + - mountPythonScript + - mountPy4j + - "-e PYTHONPATH=\"" + pythonPath + "\" " + - image + " " + - pythonInterpreter.getPythonExec() + " " + - "/_python_workdir/zeppelin_python.py"); + setPythonCommand( + "docker run -i --rm " + + mountPythonScript + + mountPy4j + + "-e PYTHONPATH=\"" + + pythonPath + + "\" " + + image + + " " + + pythonInterpreter.getPythonExec() + + " " + + "/_python_workdir/zeppelin_python.py"); restartPythonProcess(); out.clear(); return new InterpreterResult(InterpreterResult.Code.SUCCESS, "\"" + image + "\" activated"); @@ -112,7 +110,6 @@ public class PythonDockerInterpreter extends Interpreter { } } - public void setPythonCommand(String cmd) throws InterpreterException { pythonInterpreter.setPythonExec(cmd); } @@ -127,9 +124,7 @@ public class PythonDockerInterpreter extends Interpreter { } @Override - public void cancel(InterpreterContext context) { - - } + public void cancel(InterpreterContext context) {} @Override public FormType getFormType() { @@ -142,8 +137,8 @@ public class PythonDockerInterpreter extends Interpreter { } /** - * Use python interpreter's scheduler. - * To make sure %python.docker paragraph and %python paragraph runs sequentially + * Use python interpreter's scheduler. To make sure %python.docker paragraph and %python paragraph + * runs sequentially */ @Override public Scheduler getScheduler() { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java index fb4ba9c..ebab2db 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java @@ -19,6 +19,14 @@ package org.apache.zeppelin.python; import com.google.common.io.Files; import com.google.gson.Gson; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.DefaultExecutor; import org.apache.commons.exec.ExecuteException; @@ -45,19 +53,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import py4j.GatewayServer; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; - /** * Interpreter for Python, it is the first implementation of interpreter for Python, so with less - * features compared to IPythonInterpreter, but requires less prerequisites than - * IPythonInterpreter, only python installation is required. + * features compared to IPythonInterpreter, but requires less prerequisites than IPythonInterpreter, + * only python installation is required. */ public class PythonInterpreter extends Interpreter implements ExecuteResultHandler { private static final Logger LOGGER = LoggerFactory.getLogger(PythonInterpreter.class); @@ -75,7 +74,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl private long pythonPid = -1; private IPythonInterpreter iPythonInterpreter; private BaseZeppelinContext zeppelinContext; - private String condaPythonExec; // set by PythonCondaInterpreter + private String condaPythonExec; // set by PythonCondaInterpreter private boolean usePy4jAuth = false; public PythonInterpreter(Properties property) { @@ -86,9 +85,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl public void open() throws InterpreterException { // try IPythonInterpreter first iPythonInterpreter = getIPythonInterpreter(); - if (getProperty("zeppelin.python.useIPython", "true").equals("true") && - StringUtils.isEmpty( - iPythonInterpreter.checkIPythonPrerequisite(getPythonExec()))) { + if (getProperty("zeppelin.python.useIPython", "true").equals("true") + && StringUtils.isEmpty(iPythonInterpreter.checkIPythonPrerequisite(getPythonExec()))) { try { iPythonInterpreter.open(); LOGGER.info("IPython is available, Use IPythonInterpreter to replace PythonInterpreter"); @@ -130,8 +128,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl // container can also connect to this gateway server. String serverAddress = PythonUtils.getLocalIP(properties); String secret = PythonUtils.createSecret(256); - this.gatewayServer = PythonUtils.createGatewayServer(this, serverAddress, port, secret, - usePy4jAuth); + this.gatewayServer = + PythonUtils.createGatewayServer(this, serverAddress, port, secret, usePy4jAuth); gatewayServer.start(); // launch python process to connect to the gateway server in JVM side @@ -154,14 +152,15 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl if (usePy4jAuth) { env.put("PY4J_GATEWAY_SECRET", secret); } - LOGGER.info("Launching Python Process Command: " + cmd.getExecutable() + - " " + StringUtils.join(cmd.getArguments(), " ")); + LOGGER.info( + "Launching Python Process Command: " + + cmd.getExecutable() + + " " + + StringUtils.join(cmd.getArguments(), " ")); executor.execute(cmd, env, this); pythonScriptRunning.set(true); } - - private void createPythonScript() throws IOException { // set java.io.tmpdir to /tmp on MacOS, because docker can not share the /var folder which will // cause PythonDockerInterpreter fails. @@ -184,14 +183,12 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl return this.iPythonInterpreter != null; } - private void copyResourceToPythonWorkDir(String srcResourceName, - String dstFileName) throws IOException { + private void copyResourceToPythonWorkDir(String srcResourceName, String dstFileName) + throws IOException { FileOutputStream out = null; try { out = new FileOutputStream(pythonWorkDir.getAbsoluteFile() + "/" + dstFileName); - IOUtils.copy( - getClass().getClassLoader().getResourceAsStream(srcResourceName), - out); + IOUtils.copy(getClass().getClassLoader().getResourceAsStream(srcResourceName), out); } finally { if (out != null) { out.close(); @@ -261,9 +258,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl this.condaPythonExec = pythonExec; } - /** - * Request send to Python Daemon - */ + /** Request send to Python Daemon */ public class PythonInterpretRequest { public String statements; public boolean isForCompletion; @@ -335,9 +330,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl } // used by subclass such as PySparkInterpreter to set JobGroup before executing spark code - protected void preCallPython(InterpreterContext context) { - - } + protected void preCallPython(InterpreterContext context) {} // blocking call. Send python code to python process and get response protected void callPython(PythonInterpretRequest request) { @@ -366,8 +359,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl } if (!pythonScriptRunning.get()) { - return new InterpreterResult(Code.ERROR, "python process not running " - + outputStream.toString()); + return new InterpreterResult( + Code.ERROR, "python process not running " + outputStream.toString()); } outputStream.setInterpreterOutput(context.out); @@ -395,8 +388,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl if (!pythonScriptInitialized.get()) { // timeout. didn't get initialized message - errorMessage.add(new InterpreterResultMessage( - InterpreterResult.Type.TEXT, "Failed to initialize Python")); + errorMessage.add( + new InterpreterResultMessage(InterpreterResult.Type.TEXT, "Failed to initialize Python")); return new InterpreterResult(Code.ERROR, errorMessage); } @@ -457,11 +450,9 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl return 0; } - @Override - public List<InterpreterCompletion> completion(String buf, int cursor, - InterpreterContext interpreterContext) - throws InterpreterException { + public List<InterpreterCompletion> completion( + String buf, int cursor, InterpreterContext interpreterContext) throws InterpreterException { if (iPythonInterpreter != null) { return iPythonInterpreter.completion(buf, cursor, interpreterContext); } @@ -482,8 +473,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl String[] completionList = null; synchronized (statementFinishedNotifier) { long startTime = System.currentTimeMillis(); - while (statementOutput == null - && pythonScriptRunning.get()) { + while (statementOutput == null && pythonScriptRunning.get()) { try { if (System.currentTimeMillis() - startTime > MAX_TIMEOUT_SEC * 1000) { LOGGER.error("Python completion didn't have response for {}sec.", MAX_TIMEOUT_SEC); @@ -502,7 +492,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl Gson gson = new Gson(); completionList = gson.fromJson(statementOutput, String[].class); } - //end code for completion + // end code for completion if (completionList == null) { return new LinkedList<>(); } @@ -538,7 +528,6 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl if (indexOfReverseSeqPostion < completionStartPosition && indexOfReverseSeqPostion > 0) { completionStartPosition = indexOfReverseSeqPostion; } - } if (completionStartPosition == completionEndPosition) { @@ -546,8 +535,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl } else { completionStartPosition = completionEndPosition - completionStartPosition; } - resultCompletionText = completionScriptText.substring( - completionStartPosition, completionEndPosition); + resultCompletionText = + completionScriptText.substring(completionStartPosition, completionEndPosition); return resultCompletionText; } @@ -575,8 +564,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl IOUtils.toString(getClass().getClassLoader().getResourceAsStream(resourceName)); try { // Add hook explicitly, otherwise python will fail to execute the statement - InterpreterResult result = interpret(bootstrapCode + "\n" + "__zeppelin__._displayhook()", - InterpreterContext.get()); + InterpreterResult result = + interpret(bootstrapCode + "\n" + "__zeppelin__._displayhook()", InterpreterContext.get()); if (result.code() != Code.SUCCESS) { throw new IOException("Fail to run bootstrap script: " + resourceName); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java index 4fccc3c..fb812bd 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java @@ -17,6 +17,8 @@ package org.apache.zeppelin.python; +import java.io.IOException; +import java.util.Properties; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -24,13 +26,10 @@ import org.apache.zeppelin.interpreter.InterpreterResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.Properties; - /** * SQL over Pandas DataFrame interpreter for %python group - * <p> - * Match experience of %sparpk.sql over Spark DataFrame + * + * <p>Match experience of %sparpk.sql over Spark DataFrame */ public class PythonInterpreterPandasSql extends Interpreter { private static final Logger LOG = LoggerFactory.getLogger(PythonInterpreterPandasSql.class); @@ -73,9 +72,7 @@ public class PythonInterpreterPandasSql extends Interpreter { } @Override - public void cancel(InterpreterContext context) { - - } + public void cancel(InterpreterContext context) {} @Override public FormType getFormType() { @@ -86,5 +83,4 @@ public class PythonInterpreterPandasSql extends Interpreter { public int getProgress(InterpreterContext context) { return 0; } - } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/python/src/main/java/org/apache/zeppelin/python/PythonUtils.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonUtils.java b/python/src/main/java/org/apache/zeppelin/python/PythonUtils.java index 996518b..3cf2493 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonUtils.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonUtils.java @@ -17,11 +17,6 @@ package org.apache.zeppelin.python; -import org.apache.commons.codec.binary.Base64; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import py4j.GatewayServer; - import java.io.IOException; import java.net.Inet4Address; import java.net.InetAddress; @@ -29,28 +24,36 @@ import java.net.UnknownHostException; import java.security.SecureRandom; import java.util.List; import java.util.Properties; +import org.apache.commons.codec.binary.Base64; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import py4j.GatewayServer; public class PythonUtils { private static final Logger LOGGER = LoggerFactory.getLogger(PythonUtils.class); - public static GatewayServer createGatewayServer(Object entryPoint, - String serverAddress, - int port, - String secretKey, - boolean useAuth) throws IOException { - LOGGER.info("Launching GatewayServer at " + serverAddress + ":" + port + - ", useAuth: " + useAuth); + public static GatewayServer createGatewayServer( + Object entryPoint, String serverAddress, int port, String secretKey, boolean useAuth) + throws IOException { + LOGGER.info( + "Launching GatewayServer at " + serverAddress + ":" + port + ", useAuth: " + useAuth); if (useAuth) { try { - Class clz = Class.forName("py4j.GatewayServer$GatewayServerBuilder", true, - Thread.currentThread().getContextClassLoader()); + Class clz = + Class.forName( + "py4j.GatewayServer$GatewayServerBuilder", + true, + Thread.currentThread().getContextClassLoader()); Object builder = clz.getConstructor(Object.class).newInstance(entryPoint); builder.getClass().getMethod("authToken", String.class).invoke(builder, secretKey); builder.getClass().getMethod("javaPort", int.class).invoke(builder, port); - builder.getClass().getMethod("javaAddress", InetAddress.class).invoke(builder, - InetAddress.getByName(serverAddress)); - builder.getClass() + builder + .getClass() + .getMethod("javaAddress", InetAddress.class) + .invoke(builder, InetAddress.getByName(serverAddress)); + builder + .getClass() .getMethod("callbackClient", int.class, InetAddress.class, String.class) .invoke(builder, port, InetAddress.getByName(serverAddress), secretKey); return (GatewayServer) builder.getClass().getMethod("build").invoke(builder); @@ -58,7 +61,8 @@ public class PythonUtils { throw new IOException(e); } } else { - return new GatewayServer(entryPoint, + return new GatewayServer( + entryPoint, port, GatewayServer.DEFAULT_PYTHON_PORT, InetAddress.getByName(serverAddress), @@ -72,8 +76,7 @@ public class PythonUtils { public static String getLocalIP(Properties properties) { // zeppelin.python.gatewayserver_address is only for unit test on travis. // Because the FQDN would fail unit test on travis ci. - String gatewayserver_address = - properties.getProperty("zeppelin.python.gatewayserver_address"); + String gatewayserver_address = properties.getProperty("zeppelin.python.gatewayserver_address"); if (gatewayserver_address != null) { return gatewayserver_address; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java b/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java index 526784e..57b41bf 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java @@ -17,15 +17,12 @@ package org.apache.zeppelin.python; -import org.apache.zeppelin.interpreter.BaseZeppelinContext; -import org.apache.zeppelin.interpreter.InterpreterHookRegistry; - import java.util.List; import java.util.Map; +import org.apache.zeppelin.interpreter.BaseZeppelinContext; +import org.apache.zeppelin.interpreter.InterpreterHookRegistry; -/** - * ZeppelinContext for Python - */ +/** ZeppelinContext for Python */ public class PythonZeppelinContext extends BaseZeppelinContext { public PythonZeppelinContext(InterpreterHookRegistry hooks, int maxResult) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java index a51c053..81189e7 100644 --- a/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java @@ -17,6 +17,12 @@ package org.apache.zeppelin.python; +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +import java.io.IOException; +import java.util.List; import org.apache.zeppelin.display.ui.CheckBox; import org.apache.zeppelin.display.ui.Password; import org.apache.zeppelin.display.ui.Select; @@ -34,13 +40,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.util.List; - -import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; - public abstract class BasePythonInterpreterTest { protected InterpreterGroup intpGroup; @@ -52,13 +51,11 @@ public abstract class BasePythonInterpreterTest { @After public abstract void tearDown() throws InterpreterException; - @Test public void testPythonBasics() throws InterpreterException, InterruptedException, IOException { InterpreterContext context = getInterpreterContext(); - InterpreterResult result = - interpreter.interpret("import sys\nprint(sys.version[0])", context); + InterpreterResult result = interpreter.interpret("import sys\nprint(sys.version[0])", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); Thread.sleep(100); List<InterpreterResultMessage> interpreterResultMessages = @@ -145,8 +142,8 @@ public abstract class BasePythonInterpreterTest { if (interpreter instanceof IPythonInterpreter) { interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); - assertTrue(interpreterResultMessages.get(0).getData().contains( - "name 'unknown' is not defined")); + assertTrue( + interpreterResultMessages.get(0).getData().contains("name 'unknown' is not defined")); } else if (interpreter instanceof PythonInterpreter) { assertTrue(result.message().get(0).getData().contains("name 'unknown' is not defined")); } @@ -166,11 +163,13 @@ public abstract class BasePythonInterpreterTest { // ZEPPELIN-1133 context = getInterpreterContext(); - result = interpreter.interpret( - "from __future__ import print_function\n" + - "def greet(name):\n" + - " print('Hello', name)\n" + - "greet('Jack')", context); + result = + interpreter.interpret( + "from __future__ import print_function\n" + + "def greet(name):\n" + + " print('Hello', name)\n" + + "greet('Jack')", + context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); @@ -195,8 +194,8 @@ public abstract class BasePythonInterpreterTest { assertEquals(0, interpreterResultMessages.size()); context = getInterpreterContext(); - result = interpreter.interpret( - "# print('Hello')\n# print('How are u?')\n# time.sleep(1)", context); + result = + interpreter.interpret("# print('Hello')\n# print('How are u?')\n# time.sleep(1)", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); @@ -259,8 +258,7 @@ public abstract class BasePythonInterpreterTest { // Password context = getInterpreterContext(); - result = - interpreter.interpret("z.password(name='pwd_1')", context); + result = interpreter.interpret("z.password(name='pwd_1')", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertTrue(context.getGui().getForms().get("pwd_1") instanceof Password); @@ -269,8 +267,11 @@ public abstract class BasePythonInterpreterTest { // Select context = getInterpreterContext(); - result = interpreter.interpret("z.select(name='select_1'," + - " options=[('value_1', 'name_1'), ('value_2', 'name_2')])", context); + result = + interpreter.interpret( + "z.select(name='select_1'," + + " options=[('value_1', 'name_1'), ('value_2', 'name_2')])", + context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertEquals(1, context.getGui().getForms().size()); assertTrue(context.getGui().getForms().get("select_1") instanceof Select); @@ -282,8 +283,11 @@ public abstract class BasePythonInterpreterTest { // CheckBox context = getInterpreterContext(); - result = interpreter.interpret("z.checkbox(name='checkbox_1'," + - "options=[('value_1', 'name_1'), ('value_2', 'name_2')])", context); + result = + interpreter.interpret( + "z.checkbox(name='checkbox_1'," + + "options=[('value_1', 'name_1'), ('value_2', 'name_2')])", + context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertEquals(1, context.getGui().getForms().size()); assertTrue(context.getGui().getForms().get("checkbox_1") instanceof CheckBox); @@ -295,8 +299,11 @@ public abstract class BasePythonInterpreterTest { // Pandas DataFrame context = getInterpreterContext(); - result = interpreter.interpret("import pandas as pd\n" + - "df = pd.DataFrame({'id':[1,2,3], 'name':['a','b','c']})\nz.show(df)", context); + result = + interpreter.interpret( + "import pandas as pd\n" + + "df = pd.DataFrame({'id':[1,2,3], 'name':['a','b','c']})\nz.show(df)", + context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); @@ -304,21 +311,28 @@ public abstract class BasePythonInterpreterTest { assertEquals("id\tname\n1\ta\n2\tb\n3\tc\n", interpreterResultMessages.get(0).getData()); context = getInterpreterContext(); - result = interpreter.interpret("import pandas as pd\n" + - "df = pd.DataFrame({'id':[1,2,3,4], 'name':['a','b','c', 'd']})\nz.show(df)", context); + result = + interpreter.interpret( + "import pandas as pd\n" + + "df = pd.DataFrame({'id':[1,2,3,4], 'name':['a','b','c', 'd']})\nz.show(df)", + context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(2, interpreterResultMessages.size()); assertEquals(InterpreterResult.Type.TABLE, interpreterResultMessages.get(0).getType()); assertEquals("id\tname\n1\ta\n2\tb\n3\tc\n", interpreterResultMessages.get(0).getData()); assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(1).getType()); - assertEquals("<font color=red>Results are limited by 3.</font>\n", + assertEquals( + "<font color=red>Results are limited by 3.</font>\n", interpreterResultMessages.get(1).getData()); // z.show(matplotlib) context = getInterpreterContext(); - result = interpreter.interpret("import matplotlib.pyplot as plt\n" + - "data=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)\nz.show(plt)", context); + result = + interpreter.interpret( + "import matplotlib.pyplot as plt\n" + + "data=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)\nz.show(plt)", + context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); @@ -326,8 +340,11 @@ public abstract class BasePythonInterpreterTest { // clear output context = getInterpreterContext(); - result = interpreter.interpret("import time\nprint(\"Hello\")\n" + - "time.sleep(0.5)\nz.getInterpreterContext().out().clear()\nprint(\"world\")\n", context); + result = + interpreter.interpret( + "import time\nprint(\"Hello\")\n" + + "time.sleep(0.5)\nz.getInterpreterContext().out().clear()\nprint(\"world\")\n", + context); assertEquals("%text world\n", context.out.getCurrentOutput().toString()); } @@ -337,15 +354,20 @@ public abstract class BasePythonInterpreterTest { String restoreCode = "z = __zeppelin__\n"; String validCode = "z.input(\"test\")\n"; - assertEquals(InterpreterResult.Code.SUCCESS, + assertEquals( + InterpreterResult.Code.SUCCESS, interpreter.interpret(validCode, getInterpreterContext()).code()); - assertEquals(InterpreterResult.Code.SUCCESS, + assertEquals( + InterpreterResult.Code.SUCCESS, interpreter.interpret(redefinitionCode, getInterpreterContext()).code()); - assertEquals(InterpreterResult.Code.ERROR, + assertEquals( + InterpreterResult.Code.ERROR, interpreter.interpret(validCode, getInterpreterContext()).code()); - assertEquals(InterpreterResult.Code.SUCCESS, + assertEquals( + InterpreterResult.Code.SUCCESS, interpreter.interpret(restoreCode, getInterpreterContext()).code()); - assertEquals(InterpreterResult.Code.SUCCESS, + assertEquals( + InterpreterResult.Code.SUCCESS, interpreter.interpret(validCode, getInterpreterContext()).code()); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java index 4b6bfdb..5fba6ba 100644 --- a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java @@ -17,6 +17,13 @@ package org.apache.zeppelin.python; +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -26,15 +33,6 @@ import org.apache.zeppelin.interpreter.InterpreterResultMessage; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.junit.Test; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; - - public class IPythonInterpreterTest extends BasePythonInterpreterTest { protected Properties initIntpProperties() { @@ -113,8 +111,11 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest { public void testIPythonPlotting() throws InterpreterException, InterruptedException, IOException { // matplotlib InterpreterContext context = getInterpreterContext(); - InterpreterResult result = interpreter.interpret("%matplotlib inline\n" + - "import matplotlib.pyplot as plt\ndata=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)", context); + InterpreterResult result = + interpreter.interpret( + "%matplotlib inline\n" + + "import matplotlib.pyplot as plt\ndata=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)", + context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); List<InterpreterResultMessage> interpreterResultMessages = @@ -144,10 +145,13 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest { // bokeh // bokeh initialization context = getInterpreterContext(); - result = interpreter.interpret("from bokeh.io import output_notebook, show\n" + - "from bokeh.plotting import figure\n" + - "import bkzep\n" + - "output_notebook(notebook_type='zeppelin')", context); + result = + interpreter.interpret( + "from bokeh.io import output_notebook, show\n" + + "from bokeh.plotting import figure\n" + + "import bkzep\n" + + "output_notebook(notebook_type='zeppelin')", + context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); @@ -159,12 +163,15 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest { // bokeh plotting context = getInterpreterContext(); - result = interpreter.interpret("from bokeh.plotting import figure, output_file, show\n" + - "x = [1, 2, 3, 4, 5]\n" + - "y = [6, 7, 2, 4, 5]\n" + - "p = figure(title=\"simple line example\", x_axis_label='x', y_axis_label='y')\n" + - "p.line(x, y, legend=\"Temp.\", line_width=2)\n" + - "show(p)", context); + result = + interpreter.interpret( + "from bokeh.plotting import figure, output_file, show\n" + + "x = [1, 2, 3, 4, 5]\n" + + "y = [6, 7, 2, 4, 5]\n" + + "p = figure(title=\"simple line example\", x_axis_label='x', y_axis_label='y')\n" + + "p.line(x, y, legend=\"Temp.\", line_width=2)\n" + + "show(p)", + context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); @@ -176,10 +183,13 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest { // ggplot context = getInterpreterContext(); - result = interpreter.interpret("from ggplot import *\n" + - "ggplot(diamonds, aes(x='price', fill='cut')) +\\\n" + - " geom_density(alpha=0.25) +\\\n" + - " facet_wrap(\"clarity\")", context); + result = + interpreter.interpret( + "from ggplot import *\n" + + "ggplot(diamonds, aes(x='price', fill='cut')) +\\\n" + + " geom_density(alpha=0.25) +\\\n" + + " facet_wrap(\"clarity\")", + context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); @@ -234,5 +244,4 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest { result = interpreter.interpret("print('1'*3000)", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); } - } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java index a2cdac8..1a8ab03 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java @@ -17,14 +17,14 @@ package org.apache.zeppelin.python; - -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterOutput; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.io.IOException; import java.util.Arrays; @@ -33,15 +33,13 @@ import java.util.List; import java.util.Map; import java.util.Properties; import java.util.regex.Matcher; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.junit.Before; +import org.junit.Test; public class PythonCondaInterpreterTest { private PythonCondaInterpreter conda; @@ -105,14 +103,14 @@ public class PythonCondaInterpreterTest { } @Test - public void testParseCondaCommonStdout() - throws IOException, InterruptedException { + public void testParseCondaCommonStdout() throws IOException, InterruptedException { - StringBuilder sb = new StringBuilder() - .append("# comment1\n") - .append("# comment2\n") - .append("env1 /location1\n") - .append("env2 /location2\n"); + StringBuilder sb = + new StringBuilder() + .append("# comment1\n") + .append("# comment2\n") + .append("env1 /location1\n") + .append("env2 /location2\n"); Map<String, String> locationPerEnv = PythonCondaInterpreter.parseCondaCommonStdout(sb.toString()); @@ -123,20 +121,15 @@ public class PythonCondaInterpreterTest { @Test public void testGetRestArgsFromMatcher() { - Matcher m = - PythonCondaInterpreter.PATTERN_COMMAND_ENV.matcher("env remove --name test --yes"); + Matcher m = PythonCondaInterpreter.PATTERN_COMMAND_ENV.matcher("env remove --name test --yes"); m.matches(); List<String> restArgs = PythonCondaInterpreter.getRestArgsFromMatcher(m); - List<String> expected = Arrays.asList(new String[]{"remove", "--name", "test", "--yes"}); + List<String> expected = Arrays.asList(new String[] {"remove", "--name", "test", "--yes"}); assertEquals(expected, restArgs); } private InterpreterContext getInterpreterContext() { - return InterpreterContext.builder() - .setInterpreterOut(new InterpreterOutput(null)) - .build(); + return InterpreterContext.builder().setInterpreterOut(new InterpreterOutput(null)).build(); } - - } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java index 04bb414..39041f2 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java @@ -16,18 +16,6 @@ */ package org.apache.zeppelin.python; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterOutput; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -import java.io.File; -import java.util.Arrays; -import java.util.Properties; - import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.doReturn; @@ -36,6 +24,17 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import java.io.File; +import java.util.Arrays; +import java.util.Properties; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + public class PythonDockerInterpreterTest { private PythonDockerInterpreter docker; private PythonInterpreter python; @@ -77,8 +76,6 @@ public class PythonDockerInterpreterTest { } private InterpreterContext getInterpreterContext() { - return InterpreterContext.builder() - .setInterpreterOut(new InterpreterOutput(null)) - .build(); + return InterpreterContext.builder().setInterpreterOut(new InterpreterOutput(null)).build(); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java index 8326612..1fb2418 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java @@ -17,6 +17,14 @@ package org.apache.zeppelin.python; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; @@ -30,15 +38,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; - public class PythonInterpreterMatplotlibTest implements InterpreterOutputListener { private InterpreterGroup intpGroup; private PythonInterpreter python; @@ -64,10 +63,11 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene out = new InterpreterOutput(this); - context = InterpreterContext.builder() - .setInterpreterOut(out) - .setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null)) - .build(); + context = + InterpreterContext.builder() + .setInterpreterOut(out) + .setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null)) + .build(); InterpreterContext.set(context); python.open(); @@ -98,12 +98,16 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene ret = python.interpret("plt.plot([1, 2, 3])", context); ret = python.interpret("plt.show()", context); - assertEquals(new String(out.getOutputAt(0).toByteArray()), - InterpreterResult.Code.SUCCESS, ret.code()); - assertEquals(new String(out.getOutputAt(0).toByteArray()), - InterpreterResult.Type.TEXT, out.getOutputAt(0).getType()); - assertEquals(new String(out.getOutputAt(1).toByteArray()), - InterpreterResult.Type.HTML, out.getOutputAt(1).getType()); + assertEquals( + new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code()); + assertEquals( + new String(out.getOutputAt(0).toByteArray()), + InterpreterResult.Type.TEXT, + out.getOutputAt(0).getType()); + assertEquals( + new String(out.getOutputAt(1).toByteArray()), + InterpreterResult.Type.HTML, + out.getOutputAt(1).getType()); assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("data:image/png;base64")); assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("<div>")); } @@ -125,8 +129,8 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene // type to HTML. ret = python.interpret("plt.show()", context); - assertEquals(new String(out.getOutputAt(0).toByteArray()), - InterpreterResult.Code.SUCCESS, ret.code()); + assertEquals( + new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code()); assertEquals(0, ret.message().size()); // Now test that new plot is drawn. It should be identical to the @@ -173,19 +177,12 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene assertNotSame(msg1, msg2); } - @Override - public void onUpdateAll(InterpreterOutput out) { - - } + public void onUpdateAll(InterpreterOutput out) {} @Override - public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) { - - } + public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {} @Override - public void onUpdate(int index, InterpreterResultMessageOutput out) { - - } + public void onUpdate(int index, InterpreterResultMessageOutput out) {} } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java index 8f6cab2..9d4c445 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java @@ -17,6 +17,13 @@ package org.apache.zeppelin.python; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Properties; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; @@ -29,25 +36,17 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.util.Arrays; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - /** * In order for this test to work, test env must have installed: + * * <ol> - * - <li>Python</li> - * - <li>NumPy</li> - * - <li>Pandas</li> - * - <li>PandaSql</li> - * <ol> - * <p> - * To run manually on such environment, use: - * <code> + * - + * <li>Python - + * <li>NumPy - + * <li>Pandas - + * <li>PandaSql + * <ol> + * <p>To run manually on such environment, use: <code> * mvn -Dpython.test.exclude='' test -pl python -am * </code> */ @@ -70,9 +69,7 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener intpGroup = new InterpreterGroup(); out = new InterpreterOutput(this); - context = InterpreterContext.builder() - .setInterpreterOut(out) - .build(); + context = InterpreterContext.builder().setInterpreterOut(out).build(); InterpreterContext.set(context); python = new PythonInterpreter(p); @@ -84,7 +81,6 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener intpGroup.put("note", Arrays.asList(python, sql)); - // to make sure python is running. InterpreterResult ret = python.interpret("print(\"python initialized\")\n", context); assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code()); @@ -118,36 +114,40 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener public void sqlOverTestDataPrintsTable() throws IOException, InterpreterException { InterpreterResult ret; // given - //String expectedTable = "name\tage\n\nmoon\t33\n\npark\t34"; + // String expectedTable = "name\tage\n\nmoon\t33\n\npark\t34"; ret = python.interpret("import pandas as pd", context); ret = python.interpret("import numpy as np", context); // DataFrame df2 \w test data - ret = python.interpret("df2 = pd.DataFrame({ 'age' : np.array([33, 51, 51, 34]), " + - "'name' : pd.Categorical(['moon','jobs','gates','park'])})", context); + ret = + python.interpret( + "df2 = pd.DataFrame({ 'age' : np.array([33, 51, 51, 34]), " + + "'name' : pd.Categorical(['moon','jobs','gates','park'])})", + context); assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code()); - //when + // when ret = sql.interpret("select name, age from df2 where age < 40", context); - //then - assertEquals(new String(out.getOutputAt(1).toByteArray()), - InterpreterResult.Code.SUCCESS, ret.code()); - assertEquals(new String(out.getOutputAt(1).toByteArray()), Type.TABLE, - out.getOutputAt(1).getType()); + // then + assertEquals( + new String(out.getOutputAt(1).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code()); + assertEquals( + new String(out.getOutputAt(1).toByteArray()), Type.TABLE, out.getOutputAt(1).getType()); assertTrue(new String(out.getOutputAt(1).toByteArray()).indexOf("moon\t33") > 0); assertTrue(new String(out.getOutputAt(1).toByteArray()).indexOf("park\t34") > 0); - assertEquals(InterpreterResult.Code.SUCCESS, - sql.interpret("select case when name==\"aa\" then name else name end from df2", - context).code()); + assertEquals( + InterpreterResult.Code.SUCCESS, + sql.interpret("select case when name==\"aa\" then name else name end from df2", context) + .code()); } @Test public void badSqlSyntaxFails() throws IOException, InterpreterException { - //when + // when InterpreterResult ret = sql.interpret("select wrong syntax", context); - //then + // then assertNotNull("Interpreter returned 'null'", ret); assertEquals(ret.toString(), InterpreterResult.Code.ERROR, ret.code()); } @@ -168,27 +168,21 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener ret = python.interpret("z.show(df1, show_index=True)", context); // then - assertEquals(new String(out.getOutputAt(0).toByteArray()), - InterpreterResult.Code.SUCCESS, ret.code()); - assertEquals(new String(out.getOutputAt(1).toByteArray()), - Type.TABLE, out.getOutputAt(1).getType()); + assertEquals( + new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code()); + assertEquals( + new String(out.getOutputAt(1).toByteArray()), Type.TABLE, out.getOutputAt(1).getType()); assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("index_name")); assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("nan")); assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("6.7")); } @Override - public void onUpdateAll(InterpreterOutput out) { - - } + public void onUpdateAll(InterpreterOutput out) {} @Override - public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) { - - } + public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {} @Override - public void onUpdate(int index, InterpreterResultMessageOutput out) { - - } + public void onUpdate(int index, InterpreterResultMessageOutput out) {} } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java index 8748c00..10c6a8e 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java @@ -17,6 +17,16 @@ package org.apache.zeppelin.python; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -25,18 +35,6 @@ import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.junit.Test; -import java.io.IOException; -import java.util.LinkedList; -import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - - public class PythonInterpreterTest extends BasePythonInterpreterTest { @Override @@ -67,7 +65,7 @@ public class PythonInterpreterTest extends BasePythonInterpreterTest { public void testCodeCompletion() throws InterpreterException, IOException, InterruptedException { super.testCodeCompletion(); - //TODO(zjffdu) PythonInterpreter doesn't support this kind of code completion for now. + // TODO(zjffdu) PythonInterpreter doesn't support this kind of code completion for now. // completion // InterpreterContext context = getInterpreterContext(); // List<InterpreterCompletion> completions = interpreter.completion("ab", 2, context); @@ -95,7 +93,8 @@ public class PythonInterpreterTest extends BasePythonInterpreterTest { @Test public void testCancelIntp() throws InterruptedException, InterpreterException { - assertEquals(InterpreterResult.Code.SUCCESS, + assertEquals( + InterpreterResult.Code.SUCCESS, interpreter.interpret("a = 1\n", getInterpreterContext()).code()); Thread t = new Thread(new infinityPythonJob()); t.start(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java ---------------------------------------------------------------------- diff --git a/sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java b/sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java index 17da1c9..cdeeacb 100644 --- a/sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java +++ b/sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java @@ -17,6 +17,10 @@ package org.apache.zeppelin.sap; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang.StringUtils; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; @@ -27,15 +31,7 @@ import org.apache.zeppelin.sap.universe.*; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; - -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -/** - * SAP Universe interpreter for Zeppelin. - */ +/** SAP Universe interpreter for Zeppelin. */ public class UniverseInterpreter extends Interpreter { public UniverseInterpreter(Properties properties) { @@ -51,8 +47,8 @@ public class UniverseInterpreter extends Interpreter { private static final char NEWLINE = '\n'; private static final char TAB = '\t'; private static final String TABLE_MAGIC_TAG = "%table "; - private static final String EMPTY_DATA_MESSAGE = "%html\n" + - "<h4><center><b>No Data Available</b></center></h4>"; + private static final String EMPTY_DATA_MESSAGE = + "%html\n" + "<h4><center><b>No Data Available</b></center></h4>"; private static final String CONCURRENT_EXECUTION_KEY = "universe.concurrent.use"; private static final String CONCURRENT_EXECUTION_COUNT = "universe.concurrent.maxConnection"; @@ -63,10 +59,10 @@ public class UniverseInterpreter extends Interpreter { String password = getProperty("universe.password"); String apiUrl = getProperty("universe.api.url"); String authType = getProperty("universe.authType"); - final int queryTimeout = Integer.parseInt( - StringUtils.defaultIfEmpty(getProperty("universe.queryTimeout"), "7200000")); - this.client = - new UniverseClient(user, password, apiUrl, authType, queryTimeout); + final int queryTimeout = + Integer.parseInt( + StringUtils.defaultIfEmpty(getProperty("universe.queryTimeout"), "7200000")); + this.client = new UniverseClient(user, password, apiUrl, authType, queryTimeout); this.universeUtil = new UniverseUtil(); } @@ -82,8 +78,10 @@ public class UniverseInterpreter extends Interpreter { @Override public InterpreterResult interpret(String originalSt, InterpreterContext context) throws InterpreterException { - final String st = Boolean.parseBoolean(getProperty("universe.interpolation", "false")) ? - interpolate(originalSt, context.getResourcePool()) : originalSt; + final String st = + Boolean.parseBoolean(getProperty("universe.interpolation", "false")) + ? interpolate(originalSt, context.getResourcePool()) + : originalSt; try { InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); String paragraphId = context.getParagraphId(); @@ -125,7 +123,7 @@ public class UniverseInterpreter extends Interpreter { try { client.closeSession(context.getParagraphId()); } catch (Exception e) { - logger.error("Error close SAP session", e ); + logger.error("Error close SAP session", e); } } } @@ -135,7 +133,7 @@ public class UniverseInterpreter extends Interpreter { try { client.closeSession(context.getParagraphId()); } catch (Exception e) { - logger.error("Error close SAP session", e ); + logger.error("Error close SAP session", e); } } @@ -150,16 +148,15 @@ public class UniverseInterpreter extends Interpreter { } @Override - public List<InterpreterCompletion> completion(String buf, int cursor, - InterpreterContext interpreterContext) - throws InterpreterException { + public List<InterpreterCompletion> completion( + String buf, int cursor, InterpreterContext interpreterContext) throws InterpreterException { List<InterpreterCompletion> candidates = new ArrayList<>(); try { universeCompleter = createOrUpdateUniverseCompleter(interpreterContext, buf, cursor); universeCompleter.complete(buf, cursor, candidates); } catch (UniverseException e) { - logger.error("Error update completer", e ); + logger.error("Error update completer", e); } return candidates; @@ -168,9 +165,9 @@ public class UniverseInterpreter extends Interpreter { @Override public Scheduler getScheduler() { String schedulerName = UniverseInterpreter.class.getName() + this.hashCode(); - return isConcurrentExecution() ? - SchedulerFactory.singleton().createOrGetParallelScheduler(schedulerName, - getMaxConcurrentConnection()) + return isConcurrentExecution() + ? SchedulerFactory.singleton() + .createOrGetParallelScheduler(schedulerName, getMaxConcurrentConnection()) : SchedulerFactory.singleton().createOrGetFIFOScheduler(schedulerName); } @@ -212,8 +209,8 @@ public class UniverseInterpreter extends Interpreter { return str.replace(TAB, WHITESPACE).replace(NEWLINE, WHITESPACE); } - private UniverseCompleter createOrUpdateUniverseCompleter(InterpreterContext interpreterContext, - final String buf, final int cursor) + private UniverseCompleter createOrUpdateUniverseCompleter( + InterpreterContext interpreterContext, final String buf, final int cursor) throws UniverseException { final UniverseCompleter completer; if (universeCompleter == null) { @@ -224,12 +221,13 @@ public class UniverseInterpreter extends Interpreter { try { final String token = client.getToken(interpreterContext.getParagraphId()); ExecutorService executorService = Executors.newFixedThreadPool(1); - executorService.execute(new Runnable() { - @Override - public void run() { - completer.createOrUpdate(client, token, buf, cursor); - } - }); + executorService.execute( + new Runnable() { + @Override + public void run() { + completer.createOrUpdate(client, token, buf, cursor); + } + }); executorService.shutdown(); @@ -240,7 +238,7 @@ public class UniverseInterpreter extends Interpreter { try { client.closeSession(interpreterContext.getParagraphId()); } catch (Exception e) { - logger.error("Error close SAP session", e ); + logger.error("Error close SAP session", e); } } return completer;
