Repository: zeppelin Updated Branches: refs/heads/master 83602f5c2 -> ff4973d43
[ZEPPELIN-1045] Apply new mechanism to PythonInterpreter ### What is this PR for? This PR is applying new interpreter register mechanism to python interpreter. ### What type of PR is it? Improvement ### What is the Jira issue? [ZEPPELIN-1045](https://issues.apache.org/jira/browse/ZEPPELIN-1045) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Mina Lee <[email protected]> Closes #1063 from minahlee/ZEPPELIN-1045 and squashes the following commits: 66b8f73 [Mina Lee] Add zeppelin.python.maxResult property to python interpreter 5013890 [Mina Lee] Apply new mechanism to PythonInterpreter Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/ff4973d4 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/ff4973d4 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/ff4973d4 Branch: refs/heads/master Commit: ff4973d4354583fa39f7c90149f1e50bb6beebc7 Parents: 83602f5 Author: Mina Lee <[email protected]> Authored: Wed Jun 22 15:41:16 2016 -0700 Committer: Alexander Bezzubov <[email protected]> Committed: Thu Jun 23 22:16:39 2016 +0900 ---------------------------------------------------------------------- docs/interpreter/python.md | 7 +++- .../zeppelin/python/PythonInterpreter.java | 22 ++++--------- .../apache/zeppelin/python/PythonProcess.java | 2 +- python/src/main/resources/bootstrap.py | 5 ++- .../src/main/resources/interpreter-setting.json | 21 ++++++++++++ .../zeppelin/python/PythonInterpreterTest.java | 34 +++++++++----------- 6 files changed, 53 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ff4973d4/docs/interpreter/python.md ---------------------------------------------------------------------- diff --git a/docs/interpreter/python.md b/docs/interpreter/python.md index 0476c68..34134a1 100644 --- a/docs/interpreter/python.md +++ b/docs/interpreter/python.md @@ -16,12 +16,17 @@ group: manual <th>Description</th> </tr> <tr> - <td>python</td> + <td>zeppelin.python</td> <td>python</td> <td>Path of the already installed Python binary (could be python2 or python3). If python is not in your $PATH you can set the absolute directory (example : /usr/bin/python) </td> </tr> + <tr> + <td>zeppelin.python.maxResult</td> + <td>1000</td> + <td>Max number of dataframe rows to display.</td> + </tr> </table> ## Enabling Python Interpreter http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ff4973d4/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java index b1da981..9dd4ed7 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java @@ -20,7 +20,6 @@ package org.apache.zeppelin.python; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; @@ -50,27 +49,16 @@ public class PythonInterpreter extends Interpreter { public static final String BOOTSTRAP_INPUT_PY = "/bootstrap_input.py"; public static final String ZEPPELIN_PYTHON = "zeppelin.python"; public static final String DEFAULT_ZEPPELIN_PYTHON = "python"; + public static final String MAX_RESULT = "zeppelin.python.maxResult"; private Integer port; private GatewayServer gatewayServer; - private long pythonPid; private Boolean py4J = false; private InterpreterContext context; + private int maxResult; PythonProcess process = null; - static { - Interpreter.register( - "python", - "python", - PythonInterpreter.class.getName(), - new InterpreterPropertyBuilder() - .add(ZEPPELIN_PYTHON, DEFAULT_ZEPPELIN_PYTHON, - "Python directory. Default : python (assume python is in your $PATH)") - .build() - ); - } - public PythonInterpreter(Properties property) { super(property); } @@ -80,6 +68,7 @@ public class PythonInterpreter extends Interpreter { logger.info("Starting Python interpreter ....."); logger.info("Python path is set to:" + property.getProperty(ZEPPELIN_PYTHON)); + maxResult = Integer.valueOf(getProperty(MAX_RESULT)); process = getPythonProcess(); try { @@ -223,7 +212,7 @@ public class PythonInterpreter extends Interpreter { return context.getGui(); } - public Integer getPy4JPort() { + public Integer getPy4jPort() { return port; } @@ -247,4 +236,7 @@ public class PythonInterpreter extends Interpreter { return port; } + public int getMaxResult() { + return maxResult; + } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ff4973d4/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java b/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java index 364d372..348ced6 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java @@ -92,7 +92,7 @@ public class PythonProcess { String output = ""; String line; while (!(line = reader.readLine()).contains("*!?flush reader!?*")) { - logger.debug("Readed line from python shell : " + line); + logger.debug("Read line from python shell : " + line); if (line.equals("...")) { logger.warn("Syntax error ! "); output += "Syntax error ! "; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ff4973d4/python/src/main/resources/bootstrap.py ---------------------------------------------------------------------- diff --git a/python/src/main/resources/bootstrap.py b/python/src/main/resources/bootstrap.py index 4f0dc5e..04a5f53 100644 --- a/python/src/main/resources/bootstrap.py +++ b/python/src/main/resources/bootstrap.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# PYTHON 2 / 3 comptability : +# PYTHON 2 / 3 compatibility : # bootstrap.py must be runnable with Python 2 or 3 # Remove interactive mode displayhook @@ -36,7 +36,7 @@ signal.signal(signal.SIGINT, intHandler) def help(): print ('%html') print ('<h2>Python Interpreter help</h2>') - print ('<h3>Python 2 & 3 comptability</h3>') + print ('<h3>Python 2 & 3 compatibility</h3>') print ('<p>The interpreter is compatible with Python 2 & 3.<br/>') print ('To change Python version, ') print ('change in the interpreter configuration the python to the ') @@ -100,4 +100,3 @@ class PyZeppelinContext(): print (self.errorMsg) z = PyZeppelinContext("") - http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ff4973d4/python/src/main/resources/interpreter-setting.json ---------------------------------------------------------------------- diff --git a/python/src/main/resources/interpreter-setting.json b/python/src/main/resources/interpreter-setting.json new file mode 100644 index 0000000..8508bd0 --- /dev/null +++ b/python/src/main/resources/interpreter-setting.json @@ -0,0 +1,21 @@ +[ + { + "group": "python", + "name": "python", + "className": "org.apache.zeppelin.python.PythonInterpreter", + "properties": { + "zeppelin.python": { + "envName": null, + "propertyName": "zeppelin.python", + "defaultValue": "python", + "description": "Python directory. It is set to python by default.(assume python is in your $PATH)" + }, + "zeppelin.python.maxResult": { + "envName": null, + "propertyName": "zeppelin.python.maxResult", + "defaultValue": "1000", + "description": "Max number of dataframe rows to display." + } + } + } +] http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ff4973d4/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java index 2944903..dbd1346 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.python; +import static org.apache.zeppelin.python.PythonInterpreter.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -53,13 +54,17 @@ public class PythonInterpreterTest { Logger logger = LoggerFactory.getLogger(PythonProcess.class); - public static final String ZEPPELIN_PYTHON = "zeppelin.python"; - public static final String DEFAULT_ZEPPELIN_PYTHON = "python"; - PythonInterpreter pythonInterpreter = null; PythonProcess mockPythonProcess; String cmdHistory; + public static Properties getPythonTestProperties() { + Properties p = new Properties(); + p.setProperty(ZEPPELIN_PYTHON, DEFAULT_ZEPPELIN_PYTHON); + p.setProperty(MAX_RESULT, "1000"); + return p; + } + @Before public void beforeTest() { cmdHistory = ""; @@ -79,20 +84,15 @@ public class PythonInterpreterTest { logger.error("Can't initiate python process", e); } - Properties properties = new Properties(); - properties.put(ZEPPELIN_PYTHON, DEFAULT_ZEPPELIN_PYTHON); - pythonInterpreter = spy(new PythonInterpreter(properties)); + pythonInterpreter = spy(new PythonInterpreter(getPythonTestProperties())); when(pythonInterpreter.getPythonProcess()).thenReturn(mockPythonProcess); - try { when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("ImportError"); } catch (IOException e) { e.printStackTrace(); } - - } @Test @@ -111,7 +111,7 @@ public class PythonInterpreterTest { py4j JavaGateway is not running */ pythonInterpreter.open(); - assertNull(pythonInterpreter.getPy4JPort()); + assertNull(pythonInterpreter.getPy4jPort()); assertTrue(cmdHistory.contains("def help()")); assertTrue(cmdHistory.contains("class PyZeppelinContext():")); @@ -122,8 +122,7 @@ public class PythonInterpreterTest { } @Test - public void testPy4JInstalled() { - + public void testPy4jInstalled() { /* If Py4J installed, bootstrap_input.py @@ -137,7 +136,7 @@ public class PythonInterpreterTest { e.printStackTrace(); } pythonInterpreter.open(); - Integer py4jPort = pythonInterpreter.getPy4JPort(); + Integer py4jPort = pythonInterpreter.getPy4jPort(); assertNotNull(py4jPort); assertTrue(cmdHistory.contains("def help()")); @@ -147,8 +146,7 @@ public class PythonInterpreterTest { assertTrue(cmdHistory.contains("GatewayClient(port=" + py4jPort + ")")); assertTrue(cmdHistory.contains("org.apache.zeppelin.display.Input")); - - assertTrue(checkSocketAdress(py4jPort)); + assertTrue(checkSocketAddress(py4jPort)); } @@ -162,12 +160,12 @@ public class PythonInterpreterTest { e.printStackTrace(); } pythonInterpreter.open(); - Integer py4jPort = pythonInterpreter.getPy4JPort(); + Integer py4jPort = pythonInterpreter.getPy4jPort(); assertNotNull(py4jPort); pythonInterpreter.close(); - assertFalse(checkSocketAdress(py4jPort)); + assertFalse(checkSocketAddress(py4jPort)); try { verify(mockPythonProcess, times(1)).close(); } catch (IOException e) { @@ -189,7 +187,7 @@ public class PythonInterpreterTest { - private boolean checkSocketAdress(Integer py4jPort) { + private boolean checkSocketAddress(Integer py4jPort) { Socket s = new Socket(); SocketAddress sa = new InetSocketAddress("localhost", py4jPort); Boolean working = null;
