Repository: zeppelin Updated Branches: refs/heads/master 0288e4006 -> bfa84f5b8
ZEPPELIN-3734. Remove runtimeinfo from note.json ### What is this PR for? This is refactoring PR. Just remove runtimeInfo from note.json, because it doesn't make sense to store it into note.json especially when runtimeInfo contains any confidential info. Besides that this PR also remove runtimeInfo in InterpreterSetting and also some code refactoring. ### What type of PR is it? [Improvement | Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-3734 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang <[email protected]> Closes #3150 from zjffdu/ZEPPELIN-3734 and squashes the following commits: 7b3ef90a3 [Jeff Zhang] ZEPPELIN-3734. Remove runtimeinfo from note.json Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/bfa84f5b Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/bfa84f5b Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/bfa84f5b Branch: refs/heads/master Commit: bfa84f5b8d48cad4e4a4a41a3ee56ad4d6d8f3ca Parents: 0288e40 Author: Jeff Zhang <[email protected]> Authored: Mon Aug 20 10:50:24 2018 +0800 Committer: Jeff Zhang <[email protected]> Committed: Wed Aug 22 10:36:42 2018 +0800 ---------------------------------------------------------------------- .../zeppelin/rest/InterpreterRestApi.java | 1 - .../apache/zeppelin/rest/NotebookRestApi.java | 2 +- .../apache/zeppelin/socket/NotebookServer.java | 28 +-- .../apache/zeppelin/recovery/RecoveryTest.java | 2 +- .../zeppelin/rest/InterpreterRestApiTest.java | 6 +- .../zeppelin/rest/NotebookRestApiTest.java | 6 +- .../zeppelin/rest/ZeppelinSparkClusterTest.java | 68 +++--- .../zeppelin/service/NotebookServiceTest.java | 2 +- .../interpreter/InterpreterSetting.java | 34 +-- .../java/org/apache/zeppelin/notebook/Note.java | 4 +- .../org/apache/zeppelin/notebook/Notebook.java | 1 - .../org/apache/zeppelin/notebook/Paragraph.java | 213 ++++++------------- .../notebook/ParagraphWithRuntimeInfo.java | 35 +++ .../helium/HeliumApplicationFactoryTest.java | 10 +- .../apache/zeppelin/notebook/NotebookTest.java | 38 ++-- .../apache/zeppelin/notebook/ParagraphTest.java | 3 +- 16 files changed, 189 insertions(+), 264 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java index 74efa27..abf7de8 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java @@ -197,7 +197,6 @@ public class InterpreterRestApi { } else { interpreterSettingManager.restart(settingId, noteId, SecurityUtils.getPrincipal()); } - notebookServer.clearParagraphRuntimeInfo(setting); } catch (InterpreterException e) { logger.error("Exception in InterpreterRestApi while restartSetting ", e); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index bcf89ea..f831f3f 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -789,7 +789,7 @@ public class NotebookRestApi extends AbstractRestApi { new HashMap<>(), false, true, getServiceContext(), new RestServiceCallback<>())) { Note note = notebookService.getNote(noteId, getServiceContext(), new RestServiceCallback<>()); Paragraph p = note.getParagraph(paragraphId); - InterpreterResult result = p.getResult(); + InterpreterResult result = p.getReturn(); if (result.code() == InterpreterResult.Code.SUCCESS) { return new JsonResponse<>(Status.OK, result).build(); } else { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index e1efae6..35da481 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -48,6 +48,7 @@ import org.apache.zeppelin.notebook.NotebookEventListener; import org.apache.zeppelin.notebook.NotebookImportDeserializer; import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.notebook.ParagraphJobListener; +import org.apache.zeppelin.notebook.ParagraphWithRuntimeInfo; import org.apache.zeppelin.notebook.repo.NotebookRepoWithVersionControl.Revision; import org.apache.zeppelin.notebook.socket.Message; import org.apache.zeppelin.notebook.socket.Message.OP; @@ -709,7 +710,8 @@ public class NotebookServer extends WebSocketServlet if (note.isPersonalizedMode()) { broadcastParagraphs(p.getUserParagraphMap(), p); } else { - broadcast(note.getId(), new Message(OP.PARAGRAPH).put("paragraph", p)); + broadcast(note.getId(), new Message(OP.PARAGRAPH).put("paragraph", + new ParagraphWithRuntimeInfo(p))); } } @@ -2372,13 +2374,13 @@ public class NotebookServer extends WebSocketServlet if (paragraph != null) { InterpreterSetting setting = notebook().getInterpreterSettingManager() .get(interpreterSettingId); - setting.addNoteToPara(noteId, paragraphId); String label = metaInfos.get("label"); String tooltip = metaInfos.get("tooltip"); List<String> keysToRemove = Arrays.asList("noteId", "paraId", "label", "tooltip"); for (String removeKey : keysToRemove) { metaInfos.remove(removeKey); } + paragraph .updateRuntimeInfos(label, tooltip, metaInfos, setting.getGroup(), setting.getId()); broadcast( @@ -2389,28 +2391,6 @@ public class NotebookServer extends WebSocketServlet } } - public void clearParagraphRuntimeInfo(InterpreterSetting setting) { - Map<String, Set<String>> noteIdAndParaMap = setting.getNoteIdAndParaMap(); - if (noteIdAndParaMap != null && !noteIdAndParaMap.isEmpty()) { - for (String noteId : noteIdAndParaMap.keySet()) { - Set<String> paraIdSet = noteIdAndParaMap.get(noteId); - if (paraIdSet != null && !paraIdSet.isEmpty()) { - for (String paraId : paraIdSet) { - Note note = notebook().getNote(noteId); - if (note != null) { - Paragraph paragraph = note.getParagraph(paraId); - if (paragraph != null) { - paragraph.clearRuntimeInfo(setting.getId()); - broadcast(noteId, new Message(OP.PARAGRAPH).put("paragraph", paragraph)); - } - } - } - } - } - } - setting.clearNoteIdAndParaMap(); - } - private void broadcastNoteForms(Note note) { GUI formsSettings = new GUI(); formsSettings.setForms(note.getNoteForms()); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-server/src/test/java/org/apache/zeppelin/recovery/RecoveryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/recovery/RecoveryTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/recovery/RecoveryTest.java index 0336402..7a58a41 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/recovery/RecoveryTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/recovery/RecoveryTest.java @@ -90,7 +90,7 @@ public class RecoveryTest extends AbstractTestRestApi { assertEquals(resp.get("status"), "OK"); post.releaseConnection(); assertEquals(Job.Status.FINISHED, p1.getStatus()); - assertEquals("abc\n", p1.getResult().message().get(0).getData()); + assertEquals("abc\n", p1.getReturn().message().get(0).getData()); } @Test http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java index 5a0dc9f..0b3bc23 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java @@ -249,7 +249,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi { while (p.getStatus() != Status.FINISHED) { Thread.sleep(100); } - assertEquals(p.getResult().message().get(0).getData(), getSimulatedMarkdownResult("markdown")); + assertEquals(p.getReturn().message().get(0).getData(), getSimulatedMarkdownResult("markdown")); // when: restart interpreter for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterSettingManager() @@ -274,7 +274,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi { } // then - assertEquals(p.getResult().message().get(0).getData(), + assertEquals(p.getReturn().message().get(0).getData(), getSimulatedMarkdownResult("markdown restarted")); ZeppelinServer.notebook.removeNote(note.getId(), anonymous); } @@ -296,7 +296,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi { while (p.getStatus() != Status.FINISHED) { Thread.sleep(100); } - assertEquals(p.getResult().message().get(0).getData(), getSimulatedMarkdownResult("markdown")); + assertEquals(p.getReturn().message().get(0).getData(), getSimulatedMarkdownResult("markdown")); // when: get md interpreter InterpreterSetting mdIntpSetting = null; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java index 66273c8..9d96ee5 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java @@ -148,7 +148,7 @@ public class NotebookRestApiTest extends AbstractTestRestApi { assertEquals(Job.Status.FINISHED, p1.getStatus()); assertEquals(Job.Status.FINISHED, p2.getStatus()); - assertEquals("abc\n", p2.getResult().message().get(0).getData()); + assertEquals("abc\n", p2.getReturn().message().get(0).getData()); } @Test @@ -334,7 +334,7 @@ public class NotebookRestApiTest extends AbstractTestRestApi { assertEquals(Job.Status.FINISHED, p1.getStatus()); assertEquals(Job.Status.FINISHED, p2.getStatus()); - assertNotNull(p2.getResult()); - assertEquals("abc\n", p2.getResult().message().get(0).getData()); + assertNotNull(p2.getReturn()); + assertEquals("abc\n", p2.getReturn().message().get(0).getData()); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java index ef025e6..ec370d4 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java @@ -157,18 +157,18 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { assertEquals("hello\n" + "import java.util.Date\n" + "import java.net.URL\n", - p.getResult().message().get(0).getData()); + p.getReturn().message().get(0).getData()); p.setText("%spark invalid_code"); note.run(p.getId(), true); assertEquals(Status.ERROR, p.getStatus()); - assertTrue(p.getResult().message().get(0).getData().contains("error: ")); + assertTrue(p.getReturn().message().get(0).getData().contains("error: ")); // test local properties p.setText("%spark(p1=v1,p2=v2) print(z.getInterpreterContext().getLocalProperties().size())"); note.run(p.getId(), true); assertEquals(Status.FINISHED, p.getStatus()); - assertEquals("2", p.getResult().message().get(0).getData()); + assertEquals("2", p.getReturn().message().get(0).getData()); } @Test @@ -178,7 +178,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { p.setText("%spark print(sc.parallelize(1 to 10).reduce(_ + _))"); note.run(p.getId(), true); assertEquals(Status.FINISHED, p.getStatus()); - assertEquals("55", p.getResult().message().get(0).getData()); + assertEquals("55", p.getReturn().message().get(0).getData()); } @Test @@ -190,7 +190,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { "df.collect()"); note.run(p.getId(), true); assertEquals(Status.FINISHED, p.getStatus()); - assertTrue(p.getResult().message().get(0).getData().contains( + assertTrue(p.getReturn().message().get(0).getData().contains( "Array[org.apache.spark.sql.Row] = Array([hello,20])")); // test display DataFrame @@ -199,8 +199,8 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { "z.show(df)"); note.run(p.getId(), true); assertEquals(Status.FINISHED, p.getStatus()); - assertEquals(InterpreterResult.Type.TABLE, p.getResult().message().get(0).getType()); - assertEquals("_1\t_2\nhello\t20\n", p.getResult().message().get(0).getData()); + assertEquals(InterpreterResult.Type.TABLE, p.getReturn().message().get(0).getType()); + assertEquals("_1\t_2\nhello\t20\n", p.getReturn().message().get(0).getData()); // test display DataSet if (isSpark2()) { @@ -209,8 +209,8 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { "z.show(ds)"); note.run(p.getId(), true); assertEquals(Status.FINISHED, p.getStatus()); - assertEquals(InterpreterResult.Type.TABLE, p.getResult().message().get(0).getType()); - assertEquals("_1\t_2\nhello\t20\n", p.getResult().message().get(0).getData()); + assertEquals(InterpreterResult.Type.TABLE, p.getReturn().message().get(0).getType()); + assertEquals("_1\t_2\nhello\t20\n", p.getReturn().message().get(0).getData()); } } @@ -229,7 +229,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { ); note.run(p.getId(), true); assertEquals(Status.FINISHED, p.getStatus()); - assertEquals("[1] 3", p.getResult().message().get(0).getData().trim()); + assertEquals("[1] 3", p.getReturn().message().get(0).getData().trim()); } // @Test @@ -242,7 +242,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { p.setText("%spark.pyspark sc.parallelize(range(1, 11)).reduce(lambda a, b: a + b)"); note.run(p.getId(), true); assertEquals(Status.FINISHED, p.getStatus()); - assertEquals("55\n", p.getResult().message().get(0).getData()); + assertEquals("55\n", p.getReturn().message().get(0).getData()); if (!isSpark2()) { // run sqlContext test p = note.addNewParagraph(anonymous); @@ -251,7 +251,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { "df.collect()"); note.run(p.getId(), true); assertEquals(Status.FINISHED, p.getStatus()); - assertEquals("[Row(age=20, id=1)]\n", p.getResult().message().get(0).getData()); + assertEquals("[Row(age=20, id=1)]\n", p.getReturn().message().get(0).getData()); // test display Dataframe p = note.addNewParagraph(anonymous); @@ -261,9 +261,9 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { note.run(p.getId(), true); waitForFinish(p); assertEquals(Status.FINISHED, p.getStatus()); - assertEquals(InterpreterResult.Type.TABLE, p.getResult().message().get(0).getType()); + assertEquals(InterpreterResult.Type.TABLE, p.getReturn().message().get(0).getType()); // TODO(zjffdu), one more \n is appended, need to investigate why. - assertEquals("age\tid\n20\t1\n", p.getResult().message().get(0).getData()); + assertEquals("age\tid\n20\t1\n", p.getReturn().message().get(0).getData()); // test udf p = note.addNewParagraph(anonymous); @@ -271,8 +271,8 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { "sqlContext.sql(\"select f1(\\\"abc\\\") as len\").collect()"); note.run(p.getId(), true); assertEquals(Status.FINISHED, p.getStatus()); - assertTrue("[Row(len=u'3')]\n".equals(p.getResult().message().get(0).getData()) || - "[Row(len='3')]\n".equals(p.getResult().message().get(0).getData())); + assertTrue("[Row(len=u'3')]\n".equals(p.getReturn().message().get(0).getData()) || + "[Row(len='3')]\n".equals(p.getReturn().message().get(0).getData())); // test exception p = note.addNewParagraph(anonymous); @@ -285,9 +285,9 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { p.setText("%pyspark a=1\n\nprint(a2)"); note.run(p.getId(), true); assertEquals(Status.ERROR, p.getStatus()); - assertTrue(p.getResult().message().get(0).getData() + assertTrue(p.getReturn().message().get(0).getData() .contains("Fail to execute line 3: print(a2)")); - assertTrue(p.getResult().message().get(0).getData() + assertTrue(p.getReturn().message().get(0).getData() .contains("name 'a2' is not defined")); } else { // run SparkSession test @@ -297,7 +297,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { "df.collect()"); note.run(p.getId(), true); assertEquals(Status.FINISHED, p.getStatus()); - assertEquals("[Row(age=20, id=1)]\n", p.getResult().message().get(0).getData()); + assertEquals("[Row(age=20, id=1)]\n", p.getReturn().message().get(0).getData()); // test udf p = note.addNewParagraph(anonymous); @@ -306,8 +306,8 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { "spark.sql(\"select f1(\\\"abc\\\") as len\").collect()"); note.run(p.getId(), true); assertEquals(Status.FINISHED, p.getStatus()); - assertTrue("[Row(len=u'3')]\n".equals(p.getResult().message().get(0).getData()) || - "[Row(len='3')]\n".equals(p.getResult().message().get(0).getData())); + assertTrue("[Row(len=u'3')]\n".equals(p.getReturn().message().get(0).getData()) || + "[Row(len='3')]\n".equals(p.getReturn().message().get(0).getData())); } } @@ -331,7 +331,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { assertEquals(Status.FINISHED, p1.getStatus()); note.run(p2.getId(), true); assertEquals(Status.FINISHED, p2.getStatus()); - assertEquals("10", p2.getResult().message().get(0).getData()); + assertEquals("10", p2.getReturn().message().get(0).getData()); Paragraph p3 = note.addNewParagraph(anonymous); p3.setText("%spark println(new java.util.Date())"); @@ -345,7 +345,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { waitForFinish(p3); assertEquals(Status.FINISHED, p3.getStatus()); - String p3result = p3.getResult().message().get(0).getData(); + String p3result = p3.getReturn().message().get(0).getData(); assertTrue(p3result.length() > 0); // z.run(noteId, paragraphId) @@ -355,7 +355,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { note.run(p0.getId(), true); waitForFinish(p3); assertEquals(Status.FINISHED, p3.getStatus()); - assertEquals("END\n", p3.getResult().message().get(0).getData()); + assertEquals("END\n", p3.getReturn().message().get(0).getData()); // run paragraph in note2 via paragraph in note1 Note note2 = ZeppelinServer.notebook.createNote(anonymous); @@ -377,7 +377,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { waitForFinish(p21); assertEquals(Status.FINISHED, p20.getStatus()); assertEquals(Status.FINISHED, p21.getStatus()); - assertEquals("1", p21.getResult().message().get(0).getData()); + assertEquals("1", p21.getReturn().message().get(0).getData()); } @Test @@ -404,11 +404,11 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { assertEquals(Status.FINISHED, p1.getStatus()); assertEquals(Status.FINISHED, p2.getStatus()); - assertEquals("hello world\n", p2.getResult().message().get(0).getData()); + assertEquals("hello world\n", p2.getReturn().message().get(0).getData()); assertEquals(Status.FINISHED, p3.getStatus()); - assertEquals("hello world\n", p3.getResult().message().get(0).getData()); + assertEquals("hello world\n", p3.getReturn().message().get(0).getData()); assertEquals(Status.FINISHED, p4.getStatus()); - assertEquals("hello world\n", p4.getResult().message().get(0).getData()); + assertEquals("hello world\n", p4.getReturn().message().get(0).getData()); } @Test @@ -431,13 +431,13 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { assertEquals(Status.FINISHED, p1.getStatus()); assertEquals(Status.FINISHED, p2.getStatus()); - assertEquals("1\n3\n5\n4\n2\n", p2.getResult().message().get(0).getData()); + assertEquals("1\n3\n5\n4\n2\n", p2.getReturn().message().get(0).getData()); Note note2 = ZeppelinServer.notebook.createNote(anonymous); Paragraph p3 = note2.addNewParagraph(anonymous); p3.setText("%python print(6)"); note2.run(p3.getId(), true); - assertEquals("1\n6\n2\n", p3.getResult().message().get(0).getData()); + assertEquals("1\n6\n2\n", p3.getReturn().message().get(0).getData()); } @Test @@ -471,7 +471,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { note.run(p1.getId(), true); assertEquals(Status.FINISHED, p1.getStatus()); - assertEquals("2\n", p1.getResult().message().get(0).getData()); + assertEquals("2\n", p1.getReturn().message().get(0).getData()); } private void verifySparkVersionNumber() throws IOException { @@ -482,7 +482,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { note.run(p.getId()); waitForFinish(p); assertEquals(Status.FINISHED, p.getStatus()); - assertEquals(sparkVersion, p.getResult().message().get(0).getData()); + assertEquals(sparkVersion, p.getReturn().message().get(0).getData()); } private int toIntSparkVersion(String sparkVersion) { @@ -518,7 +518,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { assertEquals("my_checkbox", formIter.next()); // check dynamic forms values - String[] result = p.getResult().message().get(0).getData().split("\n"); + String[] result = p.getReturn().message().get(0).getData().split("\n"); assertEquals(5, result.length); assertEquals("default_name", result[0]); assertEquals("null", result[1]); @@ -549,7 +549,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { assertEquals("my_checkbox", formIter.next()); // check dynamic forms values - String[] result = p.getResult().message().get(0).getData().split("\n"); + String[] result = p.getReturn().message().get(0).getData().split("\n"); assertEquals(4, result.length); assertEquals("default_name", result[0]); assertEquals("None", result[1]); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-server/src/test/java/org/apache/zeppelin/service/NotebookServiceTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/service/NotebookServiceTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/service/NotebookServiceTest.java index cf3a67f..ad13dae 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/service/NotebookServiceTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/service/NotebookServiceTest.java @@ -177,7 +177,7 @@ public class NotebookServiceTest extends AbstractTestRestApi { // clean output reset(callback); notebookService.clearParagraphOutput(note1.getId(), p.getId(), context, callback); - assertNull(p.getResult()); + assertNull(p.getReturn()); verify(callback).onSuccess(p, context); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 3278e2f..36fc1f1 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -119,10 +119,6 @@ public class InterpreterSetting { private transient ApplicationEventListener appEventListener; private transient DependencyResolver dependencyResolver; - // Map of the note and paragraphs which has runtime infos generated by this interpreter setting. - // This map is used to clear the infos in paragraph when the interpretersetting is restarted - private transient Map<String, Set<String>> runtimeInfosToBeCleared; - private transient ZeppelinConfiguration conf = new ZeppelinConfiguration(); // TODO(zjffdu) ShellScriptLauncher is the only launcher implemention for now. It could be other @@ -485,7 +481,6 @@ public class InterpreterSetting { intpGroup.close(); } interpreterGroups.clear(); - this.runtimeInfosToBeCleared = null; } public void setProperties(Object object) { @@ -640,26 +635,6 @@ public class InterpreterSetting { this.interpreterRunner = interpreterRunner; } - public void addNoteToPara(String noteId, String paraId) { - if (runtimeInfosToBeCleared == null) { - runtimeInfosToBeCleared = new HashMap<>(); - } - Set<String> paraIdSet = runtimeInfosToBeCleared.get(noteId); - if (paraIdSet == null) { - paraIdSet = new HashSet<>(); - runtimeInfosToBeCleared.put(noteId, paraIdSet); - } - paraIdSet.add(paraId); - } - - public Map<String, Set<String>> getNoteIdAndParaMap() { - return runtimeInfosToBeCleared; - } - - public void clearNoteIdAndParaMap() { - runtimeInfosToBeCleared = null; - } - public String getLauncherPlugin() { if (group.equals("spark")) { return "SparkInterpreterLauncher"; @@ -668,6 +643,15 @@ public class InterpreterSetting { } } + public boolean isUserAuthorized(List<String> userAndRoles) { + if (!option.permissionIsSet()) { + return true; + } + Set<String> intersection = new HashSet<>(userAndRoles); + intersection.retainAll(option.getOwners()); + return intersection.isEmpty(); + } + //////////////////////////// IMPORTANT //////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////// http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 437b59b..61a36ab 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -464,7 +464,7 @@ public class Note implements JsonSerializable { public void clearParagraphOutputFields(Paragraph p) { p.setReturn(null, null); - p.clearRuntimeInfo(null); + p.cleanRuntimeInfos(); } public Paragraph clearPersonalizedParagraphOutput(String paragraphId, String user) { @@ -907,7 +907,7 @@ public class Note implements JsonSerializable { public void postProcessParagraphs() { for (Paragraph p : paragraphs) { - p.clearRuntimeInfos(); + p.cleanRuntimeInfos(); p.parseText(); if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 80b076f..7cf0f54 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -436,7 +436,6 @@ public class Notebook implements NoteEventListener { if (p.getDateFinished() != null && lastUpdatedDate.before(p.getDateFinished())) { lastUpdatedDate = p.getDateFinished(); } - p.clearRuntimeInfo(null); } Map<String, List<AngularObject>> savedObjects = note.getAngularObjects(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index d5bfd1f..87dc5fd 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -75,38 +75,36 @@ import com.google.common.collect.Maps; public class Paragraph extends JobWithProgressPoller<InterpreterResult> implements Cloneable, JsonSerializable { - private static Logger logger = LoggerFactory.getLogger(Paragraph.class); + private static Logger LOGGER = LoggerFactory.getLogger(Paragraph.class); private static Pattern REPL_PATTERN = Pattern.compile("(\\s*)%([\\w\\.]+)(\\(.*?\\))?.*", Pattern.DOTALL); - private transient InterpreterFactory interpreterFactory; - private transient Interpreter interpreter; - private transient Note note; - private transient AuthenticationInfo authenticationInfo; - // personalized - private transient Map<String, Paragraph> userParagraphMap = Maps.newHashMap(); - private String title; // text is composed of intpText and scriptText. private String text; - private transient String intpText; - private transient Map<String, String> localProperties = new HashMap<>(); - private transient String scriptText; private String user; private Date dateUpdated; // paragraph configs like isOpen, colWidth, etc private Map<String, Object> config = new HashMap<>(); // form and parameter settings public GUI settings = new GUI(); - private InterpreterResult results; - private Map<String, ParagraphRuntimeInfo> runtimeInfos; - - /** - * Application states in this paragraph - */ + // Application states in this paragraph private final List<ApplicationState> apps = new LinkedList<>(); + /************** Transient fields which are not serializabled into note json **************/ + private transient String intpText; + private transient String scriptText; + private transient InterpreterFactory interpreterFactory; + private transient Interpreter interpreter; + private transient Note note; + private transient AuthenticationInfo subject; + // personalized + private transient Map<String, Paragraph> userParagraphMap = new HashMap<>(); + private transient Map<String, String> localProperties = new HashMap<>(); + private transient Map<String, ParagraphRuntimeInfo> runtimeInfos = new HashMap<>(); + + @VisibleForTesting Paragraph() { super(generateId(), null); @@ -125,6 +123,21 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen this.interpreterFactory = interpreterFactory; } + // used for clone paragraph + public Paragraph(Paragraph p2) { + super(p2.getId(), null); + this.interpreterFactory = p2.interpreterFactory; + this.note = p2.note; + this.settings.setParams(Maps.newHashMap(p2.settings.getParams())); + this.settings.setForms(Maps.newLinkedHashMap(p2.settings.getForms())); + this.setConfig(Maps.newHashMap(p2.config)); + this.setAuthenticationInfo(p2.getAuthenticationInfo()); + this.title = p2.title; + this.text = p2.text; + this.results = p2.results; + setStatus(p2.getStatus()); + } + private static String generateId() { return "paragraph_" + System.currentTimeMillis() + "_" + new SecureRandom().nextInt(); } @@ -141,25 +154,12 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen } @Override - public synchronized void setResult(InterpreterResult result) { + public void setResult(InterpreterResult result) { this.results = result; } public Paragraph cloneParagraphForUser(String user) { - Paragraph p = new Paragraph(); - p.interpreterFactory = interpreterFactory; - p.note = note; - p.settings.setParams(Maps.newHashMap(settings.getParams())); - p.settings.setForms(Maps.newLinkedHashMap(settings.getForms())); - p.setConfig(Maps.newHashMap(config)); - if (getAuthenticationInfo() != null) { - p.setAuthenticationInfo(getAuthenticationInfo()); - } - p.setTitle(getTitle()); - p.setText(getText()); - p.setResult(getReturn()); - p.setStatus(Status.READY); - p.setId(getId()); + Paragraph p = new Paragraph(this); addUser(p, user); return p; } @@ -181,7 +181,6 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen } public void setText(String newText) { - // strip white space from the beginning this.text = newText; this.dateUpdated = new Date(); parseText(); @@ -228,12 +227,14 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen } public AuthenticationInfo getAuthenticationInfo() { - return authenticationInfo; + return subject; } - public void setAuthenticationInfo(AuthenticationInfo authenticationInfo) { - this.authenticationInfo = authenticationInfo; - this.user = authenticationInfo.getUser(); + public void setAuthenticationInfo(AuthenticationInfo subject) { + this.subject = subject; + if (subject != null) { + this.user = subject.getUser(); + } } public String getTitle() { @@ -279,31 +280,25 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen } public List<InterpreterCompletion> completion(String buffer, int cursor) { + setText(buffer); try { this.interpreter = getBindedInterpreter(); } catch (InterpreterNotFoundException e) { - return null; + LOGGER.debug("Unable to get completion because there's no interpreter bind to it", e); + return new ArrayList<>(); } - setText(buffer); - cursor = calculateCursorPosition(buffer, cursor); - InterpreterContext interpreterContext = getInterpreterContext(null); try { - if (this.interpreter != null) { - return this.interpreter.completion(this.scriptText, cursor, interpreterContext); - } else { - return null; - } + return this.interpreter.completion(this.scriptText, cursor, interpreterContext); } catch (InterpreterException e) { - throw new RuntimeException("Fail to get completion", e); + LOGGER.warn("Fail to get completion", e); + return new ArrayList<>(); } } public int calculateCursorPosition(String buffer, int cursor) { - // scriptText trimmed - if (this.scriptText.isEmpty()) { return 0; } @@ -319,12 +314,8 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen this.interpreterFactory = factory; } - public InterpreterResult getResult() { - return getReturn(); - } - @Override - public synchronized InterpreterResult getReturn() { + public InterpreterResult getReturn() { return results; } @@ -346,27 +337,17 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen return null; } - private boolean hasPermission(List<String> userAndRoles, List<String> intpUsersAndRoles) { - if (1 > intpUsersAndRoles.size()) { - return true; - } - Set<String> intersection = new HashSet<>(intpUsersAndRoles); - intersection.retainAll(userAndRoles); - return (intpUsersAndRoles.isEmpty() || (intersection.size() > 0)); - } - public boolean isBlankParagraph() { return Strings.isNullOrEmpty(scriptText); } public boolean execute(boolean blocking) { if (isBlankParagraph()) { - logger.info("skip to run blank paragraph. {}", getId()); + LOGGER.info("Skip to run blank paragraph. {}", getId()); setStatus(Job.Status.FINISHED); return true; } - clearRuntimeInfo(null); try { this.interpreter = getBindedInterpreter(); setStatus(Status.READY); @@ -398,11 +379,12 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen @Override protected InterpreterResult jobRun() throws Throwable { - logger.info("Run paragraph [paragraph_id: {}, interpreter: {}, note_id: {}, user: {}]", - getId(), intpText, note.getId(), authenticationInfo.getUser()); + LOGGER.info("Run paragraph [paragraph_id: {}, interpreter: {}, note_id: {}, user: {}]", + getId(), intpText, note.getId(), subject.getUser()); + this.runtimeInfos.clear(); this.interpreter = getBindedInterpreter(); if (this.interpreter == null) { - logger.error("Can not find interpreter name " + intpText); + LOGGER.error("Can not find interpreter name " + intpText); throw new RuntimeException("Can not find interpreter for " + intpText); } InterpreterSetting interpreterSetting = ((ManagedInterpreterGroup) @@ -410,12 +392,11 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen if (interpreterSetting != null) { interpreterSetting.waitForReady(); } - if (this.hasUser()) { - if (interpreterSetting != null && interpreterHasUser(interpreterSetting) - && isUserAuthorizedToAccessInterpreter(interpreterSetting.getOption()) == false) { - logger.error("{} has no permission for {} ", authenticationInfo.getUser(), intpText); - return new InterpreterResult(Code.ERROR, - authenticationInfo.getUser() + " has no permission for " + intpText); + if (this.user != null) { + if (subject != null && !interpreterSetting.isUserAuthorized(subject.getUsersAndRoles())) { + String msg = String.format("%s has no permission for %s", subject.getUser(), intpText); + LOGGER.error(msg); + return new InterpreterResult(Code.ERROR, msg); } } @@ -451,7 +432,7 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen script = Input.getSimpleQuery(note.getNoteParams(), scriptBody, true); script = Input.getSimpleQuery(settings.getParams(), script, false); } - logger.debug("RUN : " + script); + LOGGER.debug("RUN : " + script); try { InterpreterContext context = getInterpreterContext(); InterpreterContext.set(context); @@ -469,9 +450,7 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen context.out.flush(); List<InterpreterResultMessage> resultMessages = context.out.toInterpreterResultMessage(); resultMessages.addAll(ret.message()); - InterpreterResult res = new InterpreterResult(ret.code(), resultMessages); - Paragraph p = getUserParagraph(getUser()); if (null != p) { p.setResult(res); @@ -484,20 +463,6 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen } } - private boolean hasUser() { - return this.user != null; - } - - private boolean interpreterHasUser(InterpreterSetting interpreterSetting) { - return interpreterSetting.getOption().permissionIsSet() && - interpreterSetting.getOption().getOwners() != null; - } - - private boolean isUserAuthorizedToAccessInterpreter(InterpreterOption intpOpt) { - return intpOpt.permissionIsSet() && hasPermission(authenticationInfo.getUsersAndRoles(), - intpOpt.getOwners()); - } - @Override protected boolean jobAbort() { if (interpreter == null) { @@ -536,7 +501,7 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen ((ParagraphJobListener) getListener()) .onOutputUpdate(self, index, out.toInterpreterResultMessage()); } catch (IOException e) { - logger.error(e.getMessage(), e); + LOGGER.error(e.getMessage(), e); } } @@ -547,7 +512,7 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen ((ParagraphJobListener) getListener()).onOutputUpdateAll(self, messages); updateParagraphResult(messages); } catch (IOException e) { - logger.error(e.getMessage(), e); + LOGGER.error(e.getMessage(), e); } } @@ -569,12 +534,11 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen resourcePool = this.interpreter.getInterpreterGroup().getResourcePool(); } - Credentials credentials = note.getCredentials(); - if (authenticationInfo != null) { + if (subject != null) { UserCredentials userCredentials = - credentials.getUserCredentials(authenticationInfo.getUser()); - authenticationInfo.setUserCredentials(userCredentials); + credentials.getUserCredentials(subject.getUser()); + subject.setUserCredentials(userCredentials); } InterpreterContext interpreterContext = @@ -585,7 +549,7 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen .setReplName(intpText) .setParagraphTitle(title) .setParagraphText(text) - .setAuthenticationInfo(authenticationInfo) + .setAuthenticationInfo(subject) .setLocalProperties(localProperties) .setConfig(config) .setGUI(settings) @@ -617,12 +581,6 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen setException(t); } - @Override - public Object clone() throws CloneNotSupportedException { - Paragraph paraClone = (Paragraph) this.clone(); - return paraClone; - } - private String getApplicationId(HeliumPackage pkg) { return "app_" + getNote().getId() + "-" + getId() + pkg.getName().replaceAll("\\.", "_"); } @@ -694,7 +652,7 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen public void updateRuntimeInfos(String label, String tooltip, Map<String, String> infos, String group, String intpSettingId) { if (this.runtimeInfos == null) { - this.runtimeInfos = new HashMap<String, ParagraphRuntimeInfo>(); + this.runtimeInfos = new HashMap<>(); } if (infos != null) { @@ -709,42 +667,14 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen } } - /** - * Remove runtimeinfo taht were got from the setting with id settingId - * @param settingId - */ - public void clearRuntimeInfo(String settingId) { - if (settingId != null && runtimeInfos != null) { - Set<String> keys = runtimeInfos.keySet(); - if (keys.size() > 0) { - List<String> infosToRemove = new ArrayList<>(); - for (String key : keys) { - ParagraphRuntimeInfo paragraphRuntimeInfo = runtimeInfos.get(key); - if (paragraphRuntimeInfo.getInterpreterSettingId().equals(settingId)) { - infosToRemove.add(key); - } - } - if (infosToRemove.size() > 0) { - for (String info : infosToRemove) { - runtimeInfos.remove(info); - } - } - } - } else { - this.runtimeInfos = null; - } - } - - public void clearRuntimeInfos() { - if (this.runtimeInfos != null) { - this.runtimeInfos.clear(); - } - } - public Map<String, ParagraphRuntimeInfo> getRuntimeInfos() { return runtimeInfos; } + public void cleanRuntimeInfos() { + this.runtimeInfos.clear(); + } + private GUI getNoteGui() { GUI gui = new GUI(); gui.setParams(this.note.getNoteParams()); @@ -785,11 +715,9 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen if (settings != null ? !settings.equals(paragraph.settings) : paragraph.settings != null) { return false; } - if (results != null ? !results.equals(paragraph.results) : paragraph.results != null) { - return false; - } - return runtimeInfos != null ? - runtimeInfos.equals(paragraph.runtimeInfos) : paragraph.runtimeInfos == null; + + return results != null ? + results.equals(paragraph.results) : paragraph.results == null; } @@ -803,7 +731,6 @@ public class Paragraph extends JobWithProgressPoller<InterpreterResult> implemen result1 = 31 * result1 + (config != null ? config.hashCode() : 0); result1 = 31 * result1 + (settings != null ? settings.hashCode() : 0); result1 = 31 * result1 + (results != null ? results.hashCode() : 0); - result1 = 31 * result1 + (runtimeInfos != null ? runtimeInfos.hashCode() : 0); return result1; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ParagraphWithRuntimeInfo.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ParagraphWithRuntimeInfo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ParagraphWithRuntimeInfo.java new file mode 100644 index 0000000..33dce22 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ParagraphWithRuntimeInfo.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook; + +import java.util.Map; + +/** + * This class is used for broadcast Paragrapah to frontend. + * runtimeInfos will also been prapagated to frontend. + */ +public class ParagraphWithRuntimeInfo extends Paragraph { + + private Map<String, ParagraphRuntimeInfo> runtimeInfos; + + public ParagraphWithRuntimeInfo(Paragraph p) { + super(p); + this.runtimeInfos = p.getRuntimeInfos(); + } + +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 67a5947..bc6f0ec 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -113,9 +113,9 @@ public class HeliumApplicationFactoryTest extends AbstractInterpreterTest p1.setText("%mock1 job"); p1.setAuthenticationInfo(anonymous); note1.run(p1.getId()); - while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); + while(p1.isTerminated()==false || p1.getReturn()==null) Thread.yield(); - assertEquals("repl1: job", p1.getResult().message().get(0).getData()); + assertEquals("repl1: job", p1.getReturn().message().get(0).getData()); // when assertEquals(0, p1.getAllApplicationStates().size()); @@ -158,7 +158,7 @@ public class HeliumApplicationFactoryTest extends AbstractInterpreterTest p1.setText("%mock1 job"); p1.setAuthenticationInfo(anonymous); note1.run(p1.getId()); - while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); + while(p1.isTerminated()==false || p1.getReturn()==null) Thread.yield(); assertEquals(0, p1.getAllApplicationStates().size()); String appId = heliumAppFactory.loadAndRun(pkg1, p1); @@ -197,7 +197,7 @@ public class HeliumApplicationFactoryTest extends AbstractInterpreterTest p1.setText("%mock1 job"); p1.setAuthenticationInfo(anonymous); note1.run(p1.getId()); - while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); + while(p1.isTerminated()==false || p1.getReturn()==null) Thread.yield(); assertEquals(0, p1.getAllApplicationStates().size()); String appId = heliumAppFactory.loadAndRun(pkg1, p1); @@ -262,7 +262,7 @@ public class HeliumApplicationFactoryTest extends AbstractInterpreterTest p1.setText("%mock1 job"); p1.setAuthenticationInfo(anonymous); note1.run(p1.getId()); - while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); + while(p1.isTerminated()==false || p1.getReturn()==null) Thread.yield(); assertEquals(0, p1.getAllApplicationStates().size()); String appId = heliumAppFactory.loadAndRun(pkg1, p1); ApplicationState app = p1.getApplicationState(appId); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index c5a7512..ab39952 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -247,8 +247,8 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo p1.setText("%mock1 hello world"); p1.setAuthenticationInfo(anonymous); note.run(p1.getId()); - while (p1.isTerminated() == false || p1.getResult() == null) Thread.yield(); - assertEquals("repl1: hello world", p1.getResult().message().get(0).getData()); + while (p1.isTerminated() == false || p1.getReturn() == null) Thread.yield(); + assertEquals("repl1: hello world", p1.getReturn().message().get(0).getData()); // run with specific repl Paragraph p2 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); @@ -256,8 +256,8 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo p2.setText("%mock2 hello world"); p2.setAuthenticationInfo(anonymous); note.run(p2.getId()); - while (p2.isTerminated() == false || p2.getResult() == null) Thread.yield(); - assertEquals("repl2: hello world", p2.getResult().message().get(0).getData()); + while (p2.isTerminated() == false || p2.getReturn() == null) Thread.yield(); + assertEquals("repl2: hello world", p2.getReturn().message().get(0).getData()); notebook.removeNote(note.getId(), anonymous); } @@ -383,12 +383,12 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo p1.setAuthenticationInfo(anonymous); note.run(p1.getId()); - while (p1.isTerminated() == false || p1.getResult() == null) Thread.yield(); - assertEquals("repl1: hello world", p1.getResult().message().get(0).getData()); + while (p1.isTerminated() == false || p1.getReturn() == null) Thread.yield(); + assertEquals("repl1: hello world", p1.getReturn().message().get(0).getData()); // clear paragraph output/result note.clearParagraphOutput(p1.getId()); - assertNull(p1.getResult()); + assertNull(p1.getReturn()); notebook.removeNote(note.getId(), anonymous); } @@ -431,9 +431,9 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo // when note.runAll(); - assertEquals("repl1: p1", p1.getResult().message().get(0).getData()); - assertNull(p2.getResult()); - assertEquals("repl1: p3", p3.getResult().message().get(0).getData()); + assertEquals("repl1: p1", p1.getReturn().message().get(0).getData()); + assertNull(p2.getReturn()); + assertEquals("repl1: p3", p3.getReturn().message().get(0).getData()); notebook.removeNote(note.getId(), anonymous); } @@ -778,7 +778,7 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo // Test assertEquals(p.getId(), p2.getId()); assertEquals(p.getText(), p2.getText()); - assertEquals(p.getResult().message().get(0).getData(), p2.getResult().message().get(0).getData()); + assertEquals(p.getReturn().message().get(0).getData(), p2.getReturn().message().get(0).getData()); // Verify import note with subject AuthenticationInfo subject = new AuthenticationInfo("user1"); @@ -810,7 +810,7 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo // Keep same ParagraphId assertEquals(cp.getId(), p.getId()); assertEquals(cp.getText(), p.getText()); - assertEquals(cp.getResult().message().get(0).getData(), p.getResult().message().get(0).getData()); + assertEquals(cp.getReturn().message().get(0).getData(), p.getReturn().message().get(0).getData()); // Verify clone note with subject AuthenticationInfo subject = new AuthenticationInfo("user1"); @@ -1102,7 +1102,7 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo note1.run(p1.getId()); while (p1.getStatus() != Status.FINISHED) Thread.yield(); - InterpreterResult result = p1.getResult(); + InterpreterResult result = p1.getReturn(); // remove note and recreate notebook.removeNote(note1.getId(), anonymous); @@ -1113,7 +1113,7 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo note1.run(p1.getId()); while (p1.getStatus() != Status.FINISHED) Thread.yield(); - assertNotEquals(p1.getResult().message(), result.message()); + assertNotEquals(p1.getReturn().message(), result.message()); notebook.removeNote(note1.getId(), anonymous); } @@ -1139,7 +1139,7 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo while (p1.getStatus() != Status.FINISHED) Thread.yield(); while (p2.getStatus() != Status.FINISHED) Thread.yield(); - assertEquals(p1.getResult().message().get(0).getData(), p2.getResult().message().get(0).getData()); + assertEquals(p1.getReturn().message().get(0).getData(), p2.getReturn().message().get(0).getData()); // restart interpreter with per note session enabled @@ -1155,7 +1155,7 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo while (p1.getStatus() != Status.FINISHED) Thread.yield(); while (p2.getStatus() != Status.FINISHED) Thread.yield(); - assertNotEquals(p1.getResult().message(), p2.getResult().message().get(0).getData()); + assertNotEquals(p1.getReturn().message(), p2.getReturn().message().get(0).getData()); notebook.removeNote(note1.getId(), anonymous); notebook.removeNote(note2.getId(), anonymous); @@ -1183,7 +1183,7 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo while (p1.getStatus() != Status.FINISHED) Thread.yield(); while (p2.getStatus() != Status.FINISHED) Thread.yield(); - assertEquals(p1.getResult().message().get(0).getData(), p2.getResult().message().get(0).getData()); + assertEquals(p1.getReturn().message().get(0).getData(), p2.getReturn().message().get(0).getData()); // restart interpreter with scoped mode enabled for (InterpreterSetting setting : notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId())) { @@ -1198,7 +1198,7 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo while (p1.getStatus() != Status.FINISHED) Thread.yield(); while (p2.getStatus() != Status.FINISHED) Thread.yield(); - assertNotEquals(p1.getResult().message().get(0).getData(), p2.getResult().message().get(0).getData()); + assertNotEquals(p1.getReturn().message().get(0).getData(), p2.getReturn().message().get(0).getData()); // restart interpreter with isolated mode enabled for (InterpreterSetting setting : notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId())) { @@ -1213,7 +1213,7 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo while (p1.getStatus() != Status.FINISHED) Thread.yield(); while (p2.getStatus() != Status.FINISHED) Thread.yield(); - assertNotEquals(p1.getResult().message().get(0).getData(), p2.getResult().message().get(0).getData()); + assertNotEquals(p1.getReturn().message().get(0).getData(), p2.getReturn().message().get(0).getData()); notebook.removeNote(note1.getId(), anonymous); notebook.removeNote(note2.getId(), anonymous); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bfa84f5b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java index 5e5270a..609f16c 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java @@ -266,12 +266,14 @@ public class ParagraphTest extends AbstractInterpreterTest { List<InterpreterSetting> spyInterpreterSettingList = spy(Lists.<InterpreterSetting>newArrayList()); InterpreterSetting mockInterpreterSetting = mock(InterpreterSetting.class); + when(mockInterpreterGroup.getInterpreterSetting()).thenReturn(mockInterpreterSetting); InterpreterOption mockInterpreterOption = mock(InterpreterOption.class); when(mockInterpreterSetting.getOption()).thenReturn(mockInterpreterOption); when(mockInterpreterOption.permissionIsSet()).thenReturn(false); when(mockInterpreterSetting.getStatus()).thenReturn(Status.READY); when(mockInterpreterSetting.getId()).thenReturn("mock_id_1"); when(mockInterpreterSetting.getOrCreateInterpreterGroup(anyString(), anyString())).thenReturn(mockInterpreterGroup); + when(mockInterpreterSetting.isUserAuthorized(any(List.class))).thenReturn(true); spyInterpreterSettingList.add(mockInterpreterSetting); when(mockNote.getId()).thenReturn("any_id"); @@ -285,7 +287,6 @@ public class ParagraphTest extends AbstractInterpreterTest { when(mockInterpreter.interpret(anyString(), Mockito.<InterpreterContext>any())).thenReturn(mockInterpreterResult); when(mockInterpreterResult.code()).thenReturn(Code.SUCCESS); - // Actual test List<InterpreterResultMessage> result1 = Lists.newArrayList(); result1.add(new InterpreterResultMessage(Type.TEXT, "result1"));
