Repository: incubator-zeppelin Updated Branches: refs/heads/master 7a60b3355 -> 669d408dc
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/utility/IdHashes.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/utility/IdHashes.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/utility/IdHashes.java new file mode 100644 index 0000000..812dd76 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/utility/IdHashes.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook.utility; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +/** + * Generate Tiny ID. + * + * @author anthonycorbacho + * + */ +public class IdHashes { + public static final char[] DICTIONARY = new char[] {'1', '2', '3', '4', '5', '6', '7', '8', '9', + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', + 'V', 'W', 'X', 'Y', 'Z'}; + + /** + * encodes the given string into the base of the dictionary provided in the constructor. + * + * @param value the number to encode. + * @return the encoded string. + */ + public static String encode(Long value) { + + List<Character> result = new ArrayList<Character>(); + BigInteger base = new BigInteger("" + DICTIONARY.length); + int exponent = 1; + BigInteger remaining = new BigInteger(value.toString()); + while (true) { + BigInteger a = base.pow(exponent); // 16^1 = 16 + BigInteger b = remaining.mod(a); // 119 % 16 = 7 | 112 % 256 = 112 + BigInteger c = base.pow(exponent - 1); + BigInteger d = b.divide(c); + + // if d > dictionary.length, we have a problem. but BigInteger doesnt have + // a greater than method :-( hope for the best. theoretically, d is always + // an index of the dictionary! + result.add(DICTIONARY[d.intValue()]); + remaining = remaining.subtract(b); // 119 - 7 = 112 | 112 - 112 = 0 + + // finished? + if (remaining.equals(BigInteger.ZERO)) { + break; + } + + exponent++; + } + + // need to reverse it, since the start of the list contains the least significant values + StringBuffer sb = new StringBuffer(); + for (int i = result.size() - 1; i >= 0; i--) { + sb.append(result.get(i)); + } + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java new file mode 100644 index 0000000..135ffda --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.util; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +/** + * TODO(moon) : add description. + * + * @author Leemoonsoo + * + */ +public class Util { + + public static String[] split(String str, char split) { + return split(str, new String[] {String.valueOf(split)}, false); + } + + public static String[] split(String str, String[] splitters, boolean includeSplitter) { + String escapeSeq = "\"',;<%>"; + char escapeChar = '\\'; + String[] blockStart = new String[] {"\"", "'", "<%", "N_<"}; + String[] blockEnd = new String[] {"\"", "'", "%>", "N_>"}; + + return split(str, escapeSeq, escapeChar, blockStart, blockEnd, splitters, includeSplitter); + + } + + public static String[] split(String str, String escapeSeq, char escapeChar, String[] blockStart, + String[] blockEnd, String[] splitters, boolean includeSplitter) { + + List<String> splits = new ArrayList<String>(); + + String curString = ""; + + boolean escape = false; // true when escape char is found + int lastEscapeOffset = -1; + int blockStartPos = -1; + List<Integer> blockStack = new LinkedList<Integer>(); + + for (int i = 0; i < str.length(); i++) { + char c = str.charAt(i); + + // escape char detected + if (c == escapeChar && escape == false) { + escape = true; + continue; + } + + // escaped char comes + if (escape == true) { + if (escapeSeq.indexOf(c) < 0) { + curString += escapeChar; + } + curString += c; + escape = false; + lastEscapeOffset = curString.length(); + continue; + } + + if (blockStack.size() > 0) { // inside of block + curString += c; + // check multichar block + boolean multicharBlockDetected = false; + for (int b = 0; b < blockStart.length; b++) { + if (blockStartPos >= 0 + && getBlockStr(blockStart[b]).compareTo(str.substring(blockStartPos, i)) == 0) { + blockStack.remove(0); + blockStack.add(0, b); + multicharBlockDetected = true; + break; + } + } + if (multicharBlockDetected == true) { + continue; + } + + // check if current block is nestable + if (isNestedBlock(blockStart[blockStack.get(0)]) == true) { + // try to find nested block start + + if (curString.substring(lastEscapeOffset + 1).endsWith( + getBlockStr(blockStart[blockStack.get(0)])) == true) { + blockStack.add(0, blockStack.get(0)); // block is started + blockStartPos = i; + continue; + } + } + + // check if block is finishing + if (curString.substring(lastEscapeOffset + 1).endsWith( + getBlockStr(blockEnd[blockStack.get(0)]))) { + // the block closer is one of the splitters (and not nested block) + if (isNestedBlock(blockEnd[blockStack.get(0)]) == false) { + for (String splitter : splitters) { + if (splitter.compareTo(getBlockStr(blockEnd[blockStack.get(0)])) == 0) { + splits.add(curString); + if (includeSplitter == true) { + splits.add(splitter); + } + curString = ""; + lastEscapeOffset = -1; + + break; + } + } + } + blockStartPos = -1; + blockStack.remove(0); + continue; + } + + } else { // not in the block + boolean splitted = false; + for (String splitter : splitters) { + // forward check for splitter + if (splitter.compareTo( + str.substring(i, Math.min(i + splitter.length(), str.length()))) == 0) { + splits.add(curString); + if (includeSplitter == true) { + splits.add(splitter); + } + curString = ""; + lastEscapeOffset = -1; + i += splitter.length() - 1; + splitted = true; + break; + } + } + if (splitted == true) { + continue; + } + + // add char to current string + curString += c; + + // check if block is started + for (int b = 0; b < blockStart.length; b++) { + if (curString.substring(lastEscapeOffset + 1) + .endsWith(getBlockStr(blockStart[b])) == true) { + blockStack.add(0, b); // block is started + blockStartPos = i; + break; + } + } + } + } + if (curString.length() > 0) { + splits.add(curString.trim()); + } + return splits.toArray(new String[] {}); + + } + + private static String getBlockStr(String blockDef) { + if (blockDef.startsWith("N_")) { + return blockDef.substring("N_".length()); + } else { + return blockDef; + } + } + + private static boolean isNestedBlock(String blockDef) { + if (blockDef.startsWith("N_")) { + return true; + } else { + return false; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/main/resources/exec.erb ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/resources/exec.erb b/zeppelin-zengine/src/main/resources/exec.erb deleted file mode 100644 index dc3877e..0000000 --- a/zeppelin-zengine/src/main/resources/exec.erb +++ /dev/null @@ -1,15 +0,0 @@ -<html> - <head> - </head> - <body> - -<% if z and z.result != nil %> - <% if z.result.rows != nil %> -<pre><% z.result.rows.each do |row| %> -<% row.each do |cell| %><%= cell %><% end %><% end %> -<% end %></pre> - -<% end %> - - </body> -</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/main/resources/table.erb ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/resources/table.erb b/zeppelin-zengine/src/main/resources/table.erb deleted file mode 100644 index 188902e..0000000 --- a/zeppelin-zengine/src/main/resources/table.erb +++ /dev/null @@ -1,36 +0,0 @@ -<html> - <head> - <script type='text/javascript' src='https://www.google.com/jsapi'></script> - <script type='text/javascript'> - google.load('visualization', '1', {packages:['table']}); - google.setOnLoadCallback(drawTable); - function drawTable() { - var data = new google.visualization.DataTable(); -<% if z and z.result != nil %> - <% z.result.getColumnDef.each do |col| %> - data.addColumn('string', '<%=col.name%>'); - <% end %> - <% if z.result.rows != nil %> - data.addRows([ - <% z.result.rows.each do |row| %> - [ - <% row.each do |cell| %> - '<%= cell.to_s().gsub("'","\\\\'") %>', - <% end %> - ], - <% end %> - ]); - - <% end %> -<% end %> - var table = new google.visualization.Table(document.getElementById('table_div')); - table.draw(data, {showRowNumber: true}); - } - </script> - </head> - <body> - <div id="table_div"></div> - </body> -</html> - - http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/interpreter/InterpreterFactoryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/interpreter/InterpreterFactoryTest.java deleted file mode 100644 index dbb65ea..0000000 --- a/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/interpreter/InterpreterFactoryTest.java +++ /dev/null @@ -1,112 +0,0 @@ -package com.nflabs.zeppelin.interpreter; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import java.util.Properties; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.nflabs.zeppelin.conf.ZeppelinConfiguration; -import com.nflabs.zeppelin.conf.ZeppelinConfiguration.ConfVars; -import com.nflabs.zeppelin.interpreter.mock.MockInterpreter1; -import com.nflabs.zeppelin.interpreter.mock.MockInterpreter2; - -public class InterpreterFactoryTest { - - private InterpreterFactory factory; - private File tmpDir; - private ZeppelinConfiguration conf; - private InterpreterContext context; - - @Before - public void setUp() throws Exception { - tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); - tmpDir.mkdirs(); - new File(tmpDir, "conf").mkdirs(); - - MockInterpreter1.register("mock1", "com.nflabs.zeppelin.interpreter.mock.MockInterpreter1"); - MockInterpreter2.register("mock2", "com.nflabs.zeppelin.interpreter.mock.MockInterpreter2"); - - System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath()); - System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "com.nflabs.zeppelin.interpreter.mock.MockInterpreter1,com.nflabs.zeppelin.interpreter.mock.MockInterpreter2"); - conf = new ZeppelinConfiguration(); - factory = new InterpreterFactory(conf, new InterpreterOption(false)); - context = new InterpreterContext("id", "title", "text", null, null); - - } - - @After - public void tearDown() throws Exception { - delete(tmpDir); - } - - private void delete(File file){ - if(file.isFile()) file.delete(); - else if(file.isDirectory()){ - File [] files = file.listFiles(); - if(files!=null && files.length>0){ - for(File f : files){ - delete(f); - } - } - file.delete(); - } - } - - @Test - public void testBasic() { - List<String> all = factory.getDefaultInterpreterSettingList(); - - // get interpreter - Interpreter repl1 = factory.get(all.get(0)).getInterpreterGroup().getFirst(); - assertFalse(((LazyOpenInterpreter) repl1).isOpen()); - repl1.interpret("repl1", context); - assertTrue(((LazyOpenInterpreter) repl1).isOpen()); - - // try to get unavailable interpreter - assertNull(factory.get("unknown")); - - // restart interpreter - factory.restart(all.get(0)); - repl1 = factory.get(all.get(0)).getInterpreterGroup().getFirst(); - assertFalse(((LazyOpenInterpreter) repl1).isOpen()); - } - - @Test - public void testFactoryDefaultList() throws InterpreterException, IOException { - // get default list from default setting - List<String> all = factory.getDefaultInterpreterSettingList(); - assertEquals(2, all.size()); - assertEquals(factory.get(all.get(0)).getInterpreterGroup().getFirst().getClassName(), "com.nflabs.zeppelin.interpreter.mock.MockInterpreter1"); - - // add setting - factory.add("a mock", "mock2", new InterpreterOption(false), new Properties()); - all = factory.getDefaultInterpreterSettingList(); - assertEquals(2, all.size()); - assertEquals("mock1", factory.get(all.get(0)).getName()); - assertEquals("a mock", factory.get(all.get(1)).getName()); - } - - @Test - public void testSaveLoad() throws InterpreterException, IOException { - // interpreter settings - assertEquals(2, factory.get().size()); - - // check if file saved - assertTrue(new File(conf.getInterpreterSettingPath()).exists()); - - factory.add("newsetting", "mock1", new InterpreterOption(false), new Properties()); - assertEquals(3, factory.get().size()); - - InterpreterFactory factory2 = new InterpreterFactory(conf); - assertEquals(3, factory2.get().size()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/interpreter/mock/MockInterpreter1.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/interpreter/mock/MockInterpreter1.java b/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/interpreter/mock/MockInterpreter1.java deleted file mode 100644 index dfdfea7..0000000 --- a/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/interpreter/mock/MockInterpreter1.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.nflabs.zeppelin.interpreter.mock; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import com.nflabs.zeppelin.interpreter.Interpreter; -import com.nflabs.zeppelin.interpreter.InterpreterContext; -import com.nflabs.zeppelin.interpreter.InterpreterResult; -import com.nflabs.zeppelin.scheduler.Scheduler; -import com.nflabs.zeppelin.scheduler.SchedulerFactory; - -public class MockInterpreter1 extends Interpreter{ - Map<String, Object> vars = new HashMap<String, Object>(); - - public MockInterpreter1(Properties property) { - super(property); - } - - @Override - public void open() { - } - - @Override - public void close() { - } - - @Override - public InterpreterResult interpret(String st, InterpreterContext context) { - return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: "+st); - } - - @Override - public void cancel(InterpreterContext context) { - } - - @Override - public FormType getFormType() { - return FormType.SIMPLE; - } - - @Override - public int getProgress(InterpreterContext context) { - return 0; - } - - @Override - public Scheduler getScheduler() { - return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode()); - } - - @Override - public List<String> completion(String buf, int cursor) { - return null; - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/interpreter/mock/MockInterpreter2.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/interpreter/mock/MockInterpreter2.java b/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/interpreter/mock/MockInterpreter2.java deleted file mode 100644 index c5db654..0000000 --- a/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/interpreter/mock/MockInterpreter2.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.nflabs.zeppelin.interpreter.mock; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import com.nflabs.zeppelin.interpreter.Interpreter; -import com.nflabs.zeppelin.interpreter.InterpreterContext; -import com.nflabs.zeppelin.interpreter.InterpreterResult; -import com.nflabs.zeppelin.scheduler.Scheduler; -import com.nflabs.zeppelin.scheduler.SchedulerFactory; - -public class MockInterpreter2 extends Interpreter{ - Map<String, Object> vars = new HashMap<String, Object>(); - - public MockInterpreter2(Properties property) { - super(property); - } - - @Override - public void open() { - } - - @Override - public void close() { - } - - @Override - public InterpreterResult interpret(String st, InterpreterContext context) { - return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl2: "+st); - } - - @Override - public void cancel(InterpreterContext context) { - } - - @Override - public FormType getFormType() { - return FormType.SIMPLE; - } - - @Override - public int getProgress(InterpreterContext context) { - return 0; - } - - @Override - public Scheduler getScheduler() { - return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode()); - } - - @Override - public List<String> completion(String buf, int cursor) { - return null; - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/notebook/NotebookTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/notebook/NotebookTest.java deleted file mode 100644 index 0539b08..0000000 --- a/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/notebook/NotebookTest.java +++ /dev/null @@ -1,173 +0,0 @@ -package com.nflabs.zeppelin.notebook; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -import java.io.File; -import java.io.IOException; -import java.util.Date; -import java.util.Map; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.quartz.SchedulerException; - -import com.nflabs.zeppelin.conf.ZeppelinConfiguration; -import com.nflabs.zeppelin.conf.ZeppelinConfiguration.ConfVars; -import com.nflabs.zeppelin.interpreter.InterpreterFactory; -import com.nflabs.zeppelin.interpreter.InterpreterOption; -import com.nflabs.zeppelin.interpreter.mock.MockInterpreter1; -import com.nflabs.zeppelin.interpreter.mock.MockInterpreter2; -import com.nflabs.zeppelin.scheduler.Job; -import com.nflabs.zeppelin.scheduler.Job.Status; -import com.nflabs.zeppelin.scheduler.JobListener; -import com.nflabs.zeppelin.scheduler.SchedulerFactory; - -public class NotebookTest implements JobListenerFactory{ - - private File tmpDir; - private ZeppelinConfiguration conf; - private SchedulerFactory schedulerFactory; - private File notebookDir; - private Notebook notebook; - private InterpreterFactory factory; - - @Before - public void setUp() throws Exception { - tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); - tmpDir.mkdirs(); - new File(tmpDir, "conf").mkdirs(); - notebookDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()+"/notebook"); - notebookDir.mkdirs(); - - System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath()); - System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); - System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "com.nflabs.zeppelin.interpreter.mock.MockInterpreter1,com.nflabs.zeppelin.interpreter.mock.MockInterpreter2"); - - conf = ZeppelinConfiguration.create(); - - this.schedulerFactory = new SchedulerFactory(); - - MockInterpreter1.register("mock1", "com.nflabs.zeppelin.interpreter.mock.MockInterpreter1"); - MockInterpreter2.register("mock2", "com.nflabs.zeppelin.interpreter.mock.MockInterpreter2"); - - factory = new InterpreterFactory(conf, new InterpreterOption(false)); - - notebook = new Notebook(conf, schedulerFactory, factory, this); - } - - @After - public void tearDown() throws Exception { - delete(tmpDir); - } - - @Test - public void testSelectingReplImplementation() throws IOException { - Note note = notebook.createNote(); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); - - // run with defatul repl - Paragraph p1 = note.addParagraph(); - p1.setText("hello world"); - note.run(p1.getId()); - while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); - assertEquals("repl1: hello world", p1.getResult().message()); - - // run with specific repl - Paragraph p2 = note.addParagraph(); - p2.setText("%mock2 hello world"); - note.run(p2.getId()); - while(p2.isTerminated()==false || p2.getResult()==null) Thread.yield(); - assertEquals("repl2: hello world", p2.getResult().message()); - } - - @Test - public void testPersist() throws IOException, SchedulerException{ - Note note = notebook.createNote(); - - // run with default repl - Paragraph p1 = note.addParagraph(); - p1.setText("hello world"); - note.persist(); - - Notebook notebook2 = new Notebook(conf, schedulerFactory, new InterpreterFactory(conf), this); - assertEquals(1, notebook2.getAllNotes().size()); - } - - @Test - public void testRunAll() throws IOException { - Note note = notebook.createNote(); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); - - Paragraph p1 = note.addParagraph(); - p1.setText("p1"); - Paragraph p2 = note.addParagraph(); - p2.setText("p2"); - assertEquals(null, p2.getResult()); - note.runAll(); - - while(p2.isTerminated()==false || p2.getResult()==null) Thread.yield(); - assertEquals("repl1: p2", p2.getResult().message()); - } - - @Test - public void testSchedule() throws InterruptedException, IOException{ - // create a note and a paragraph - Note note = notebook.createNote(); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); - - Paragraph p = note.addParagraph(); - p.setText("p1"); - Date dateFinished = p.getDateFinished(); - assertNull(dateFinished); - - // set cron scheduler, once a second - Map<String, Object> config = note.getConfig(); - config.put("cron", "* * * * * ?"); - note.setConfig(config); - notebook.refreshCron(note.id()); - Thread.sleep(1*1000); - dateFinished = p.getDateFinished(); - assertNotNull(dateFinished); - - // remove cron scheduler. - config.put("cron", null); - note.setConfig(config); - notebook.refreshCron(note.id()); - Thread.sleep(1*1000); - assertEquals(dateFinished, p.getDateFinished()); - } - - private void delete(File file){ - if(file.isFile()) file.delete(); - else if(file.isDirectory()){ - File [] files = file.listFiles(); - if(files!=null && files.length>0){ - for(File f : files){ - delete(f); - } - } - file.delete(); - } - } - - @Override - public JobListener getParagraphJobListener(Note note) { - return new JobListener(){ - - @Override - public void onProgressUpdate(Job job, int progress) { - } - - @Override - public void beforeStatusChange(Job job, Status before, Status after) { - } - - @Override - public void afterStatusChange(Job job, Status before, Status after) { - } - }; - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/util/UtilTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/util/UtilTest.java b/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/util/UtilTest.java deleted file mode 100644 index 715157d..0000000 --- a/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/util/UtilTest.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.nflabs.zeppelin.util; - -import junit.framework.TestCase; - -public class UtilTest extends TestCase { - - protected void setUp() throws Exception { - super.setUp(); - } - - protected void tearDown() throws Exception { - super.tearDown(); - } - - public void testSplitIncludingToken() { - String[] token = Util.split("hello | \"world '>|hehe\" > next >> sink", new String[]{"|", ">>", ">"}, true); - assertEquals(7, token.length); - assertEquals(" \"world '>|hehe\" ", token[2]); - } - - public void testSplitExcludingToken() { - String[] token = Util.split("hello | \"world '>|hehe\" > next >> sink", new String[]{"|", ">>", ">"}, false); - assertEquals(4, token.length); - assertEquals(" \"world '>|hehe\" ", token[1]); - } - - public void testSplitWithSemicolonEnd(){ - String[] token = Util.split("show tables;", ';'); - assertEquals(1, token.length); - assertEquals("show tables", token[0]); - } - - public void testEscapeTemplate(){ - String[] token = Util.split("select * from <%=table%> limit 1 > output", '>'); - assertEquals(2, token.length); - assertEquals("output", token[1]); - } - - public void testSplit(){ - String [] op = new String[]{";", "|", ">>", ">"}; - - String str = "CREATE external table news20b_train (\n"+ - " rowid int,\n"+ - " label int,\n"+ - " features ARRAY<STRING>\n"+ - ")\n"+ - "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' \n"+ - "COLLECTION ITEMS TERMINATED BY \",\" \n"+ - "STORED AS TEXTFILE;\n"; - Util.split(str, op, true); - - } - - public void testSplitDifferentBlockStartEnd(){ - String [] op = new String[]{";", "|", ">>", ">"}; - String escapeSeq = "\"',;<%>!"; - char escapeChar = '\\'; - String [] blockStart = new String[]{ "\"", "'", "<%", "<", "!"}; - String [] blockEnd = new String[]{ "\"", "'", "%>", ">", ";" }; - String [] t = Util.split("!echo a;!echo b;", escapeSeq, escapeChar, blockStart, blockEnd, op, true); - assertEquals(4, t.length); - assertEquals("!echo a;", t[0]); - assertEquals(";", t[1]); - assertEquals("!echo b;", t[2]); - assertEquals(";", t[3]); - } - - public void testNestedBlock(){ - String [] op = new String[]{";", "|", ">>", ">"}; - String escapeSeq = "\"',;<%>!"; - char escapeChar = '\\'; - String [] blockStart = new String[]{ "\"", "'", "<%", "N_<", "<", "!"}; - String [] blockEnd = new String[]{ "\"", "'", "%>", "N_>", ";", ";" }; - String [] t = Util.split("array <STRUCT<STRING>> tags|aa", escapeSeq, escapeChar, blockStart, blockEnd, op, true); - assertEquals(3, t.length); - assertEquals("array <STRUCT<STRING>> tags", t[0]); - assertEquals("aa", t[2]); - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/util/UtilsForTests.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/util/UtilsForTests.java b/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/util/UtilsForTests.java deleted file mode 100644 index dab7138..0000000 --- a/zeppelin-zengine/src/test/java/com/nflabs/zeppelin/util/UtilsForTests.java +++ /dev/null @@ -1,104 +0,0 @@ -package com.nflabs.zeppelin.util; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import com.nflabs.zeppelin.conf.ZeppelinConfiguration; - -public class UtilsForTests { - - public static File createTmpDir() throws Exception { - File tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); - tmpDir.mkdir(); - return tmpDir; - - } - /* - private static final String HADOOP_DIST="http://apache.mirror.cdnetworks.com/hadoop/common/hadoop-1.2.1/hadoop-1.2.1-bin.tar.gz";; - //private static final String HADOOP_DIST="http://www.us.apache.org/dist/hadoop/common/hadoop-1.2.1/hadoop-1.2.1-bin.tar.gz";; - - public static void getHadoop() throws MalformedURLException, IOException{ - setEnv("HADOOP_HOME", new File("./target/hadoop-1.2.1").getAbsolutePath()); - if(new File("./target/hadoop-1.2.1").isDirectory()) return; - //System.out.println("Downloading a hadoop distribution ... it will take a while"); - //FileUtils.copyURLToFile(new URL(HADOOP_DIST), new File("/tmp/zp_test_hadoop-bin.tar.gz")); - System.out.println("Unarchive hadoop distribution ... "); - new File("./target").mkdir(); - Runtime.getRuntime().exec("tar -xzf /tmp/zp_test_hadoop-bin.tar.gz -C ./target"); - } - */ - - public static void delete(File file){ - if(file.isFile()) file.delete(); - else if(file.isDirectory()){ - File [] files = file.listFiles(); - if(files!=null && files.length>0){ - for(File f : files){ - delete(f); - } - } - file.delete(); - } - } - - /** - * Utility method to create a file (if does not exist) and populate it the the given content - * - * @param path to file - * @param content of the file - * @throws IOException - */ - public static void createFileWithContent(String path, String content) throws IOException { - File f = new File(path); - if (!f.exists()) { - stringToFile(content, f); - } - } - - public static void stringToFile(String string, File file) throws IOException{ - FileOutputStream out = new FileOutputStream(file); - out.write(string.getBytes()); - out.close(); - } - - @SuppressWarnings({ "unchecked", "rawtypes" }) - public static void setEnv(String k, String v) { - Map<String, String> newenv = new HashMap<String, String>(); - newenv.put(k, v); - try { - Class<?> processEnvironmentClass = Class.forName("java.lang.ProcessEnvironment"); - Field theEnvironmentField = processEnvironmentClass.getDeclaredField("theEnvironment"); - theEnvironmentField.setAccessible(true); - Map<String, String> env = (Map<String, String>) theEnvironmentField.get(null); - env.putAll(newenv); - Field theCaseInsensitiveEnvironmentField = processEnvironmentClass.getDeclaredField("theCaseInsensitiveEnvironment"); - theCaseInsensitiveEnvironmentField.setAccessible(true); - Map<String, String> cienv = (Map<String, String>) theCaseInsensitiveEnvironmentField.get(null); - cienv.putAll(newenv); - } catch (NoSuchFieldException e) { - try { - Class[] classes = Collections.class.getDeclaredClasses(); - Map<String, String> env = System.getenv(); - for(Class cl : classes) { - if("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { - Field field = cl.getDeclaredField("m"); - field.setAccessible(true); - Object obj = field.get(env); - Map<String, String> map = (Map<String, String>) obj; - map.clear(); - map.putAll(newenv); - } - } - } catch (Exception e2) { - e2.printStackTrace(); - } - } catch (Exception e1) { - e1.printStackTrace(); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java new file mode 100644 index 0000000..5199300 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.interpreter.mock.MockInterpreter1; +import org.apache.zeppelin.interpreter.mock.MockInterpreter2; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class InterpreterFactoryTest { + + private InterpreterFactory factory; + private File tmpDir; + private ZeppelinConfiguration conf; + private InterpreterContext context; + + @Before + public void setUp() throws Exception { + tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); + tmpDir.mkdirs(); + new File(tmpDir, "conf").mkdirs(); + + MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); + MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); + + System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2"); + conf = new ZeppelinConfiguration(); + factory = new InterpreterFactory(conf, new InterpreterOption(false)); + context = new InterpreterContext("id", "title", "text", null, null); + + } + + @After + public void tearDown() throws Exception { + delete(tmpDir); + } + + private void delete(File file){ + if(file.isFile()) file.delete(); + else if(file.isDirectory()){ + File [] files = file.listFiles(); + if(files!=null && files.length>0){ + for(File f : files){ + delete(f); + } + } + file.delete(); + } + } + + @Test + public void testBasic() { + List<String> all = factory.getDefaultInterpreterSettingList(); + + // get interpreter + Interpreter repl1 = factory.get(all.get(0)).getInterpreterGroup().getFirst(); + assertFalse(((LazyOpenInterpreter) repl1).isOpen()); + repl1.interpret("repl1", context); + assertTrue(((LazyOpenInterpreter) repl1).isOpen()); + + // try to get unavailable interpreter + assertNull(factory.get("unknown")); + + // restart interpreter + factory.restart(all.get(0)); + repl1 = factory.get(all.get(0)).getInterpreterGroup().getFirst(); + assertFalse(((LazyOpenInterpreter) repl1).isOpen()); + } + + @Test + public void testFactoryDefaultList() throws InterpreterException, IOException { + // get default list from default setting + List<String> all = factory.getDefaultInterpreterSettingList(); + assertEquals(2, all.size()); + assertEquals(factory.get(all.get(0)).getInterpreterGroup().getFirst().getClassName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); + + // add setting + factory.add("a mock", "mock2", new InterpreterOption(false), new Properties()); + all = factory.getDefaultInterpreterSettingList(); + assertEquals(2, all.size()); + assertEquals("mock1", factory.get(all.get(0)).getName()); + assertEquals("a mock", factory.get(all.get(1)).getName()); + } + + @Test + public void testSaveLoad() throws InterpreterException, IOException { + // interpreter settings + assertEquals(2, factory.get().size()); + + // check if file saved + assertTrue(new File(conf.getInterpreterSettingPath()).exists()); + + factory.add("newsetting", "mock1", new InterpreterOption(false), new Properties()); + assertEquals(3, factory.get().size()); + + InterpreterFactory factory2 = new InterpreterFactory(conf); + assertEquals(3, factory2.get().size()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java new file mode 100644 index 0000000..09259b1 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.mock; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; + +public class MockInterpreter1 extends Interpreter{ + Map<String, Object> vars = new HashMap<String, Object>(); + + public MockInterpreter1(Properties property) { + super(property); + } + + @Override + public void open() { + } + + @Override + public void close() { + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: "+st); + } + + @Override + public void cancel(InterpreterContext context) { + } + + @Override + public FormType getFormType() { + return FormType.SIMPLE; + } + + @Override + public int getProgress(InterpreterContext context) { + return 0; + } + + @Override + public Scheduler getScheduler() { + return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode()); + } + + @Override + public List<String> completion(String buf, int cursor) { + return null; + } +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java new file mode 100644 index 0000000..dd465a5 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.mock; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.scheduler.SchedulerFactory; + +public class MockInterpreter2 extends Interpreter{ + Map<String, Object> vars = new HashMap<String, Object>(); + + public MockInterpreter2(Properties property) { + super(property); + } + + @Override + public void open() { + } + + @Override + public void close() { + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl2: "+st); + } + + @Override + public void cancel(InterpreterContext context) { + } + + @Override + public FormType getFormType() { + return FormType.SIMPLE; + } + + @Override + public int getProgress(InterpreterContext context) { + return 0; + } + + @Override + public Scheduler getScheduler() { + return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode()); + } + + @Override + public List<String> completion(String buf, int cursor) { + return null; + } +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java new file mode 100644 index 0000000..88af541 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.notebook; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import java.io.File; +import java.io.IOException; +import java.util.Date; +import java.util.Map; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.interpreter.mock.MockInterpreter1; +import org.apache.zeppelin.interpreter.mock.MockInterpreter2; +import org.apache.zeppelin.scheduler.Job; +import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.scheduler.JobListener; +import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.quartz.SchedulerException; + +public class NotebookTest implements JobListenerFactory{ + + private File tmpDir; + private ZeppelinConfiguration conf; + private SchedulerFactory schedulerFactory; + private File notebookDir; + private Notebook notebook; + private InterpreterFactory factory; + + @Before + public void setUp() throws Exception { + tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); + tmpDir.mkdirs(); + new File(tmpDir, "conf").mkdirs(); + notebookDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()+"/notebook"); + notebookDir.mkdirs(); + + System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); + System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2"); + + conf = ZeppelinConfiguration.create(); + + this.schedulerFactory = new SchedulerFactory(); + + MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); + MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); + + factory = new InterpreterFactory(conf, new InterpreterOption(false)); + + notebook = new Notebook(conf, schedulerFactory, factory, this); + } + + @After + public void tearDown() throws Exception { + delete(tmpDir); + } + + @Test + public void testSelectingReplImplementation() throws IOException { + Note note = notebook.createNote(); + note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + + // run with defatul repl + Paragraph p1 = note.addParagraph(); + p1.setText("hello world"); + note.run(p1.getId()); + while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); + assertEquals("repl1: hello world", p1.getResult().message()); + + // run with specific repl + Paragraph p2 = note.addParagraph(); + p2.setText("%mock2 hello world"); + note.run(p2.getId()); + while(p2.isTerminated()==false || p2.getResult()==null) Thread.yield(); + assertEquals("repl2: hello world", p2.getResult().message()); + } + + @Test + public void testPersist() throws IOException, SchedulerException{ + Note note = notebook.createNote(); + + // run with default repl + Paragraph p1 = note.addParagraph(); + p1.setText("hello world"); + note.persist(); + + Notebook notebook2 = new Notebook(conf, schedulerFactory, new InterpreterFactory(conf), this); + assertEquals(1, notebook2.getAllNotes().size()); + } + + @Test + public void testRunAll() throws IOException { + Note note = notebook.createNote(); + note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + + Paragraph p1 = note.addParagraph(); + p1.setText("p1"); + Paragraph p2 = note.addParagraph(); + p2.setText("p2"); + assertEquals(null, p2.getResult()); + note.runAll(); + + while(p2.isTerminated()==false || p2.getResult()==null) Thread.yield(); + assertEquals("repl1: p2", p2.getResult().message()); + } + + @Test + public void testSchedule() throws InterruptedException, IOException{ + // create a note and a paragraph + Note note = notebook.createNote(); + note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + + Paragraph p = note.addParagraph(); + p.setText("p1"); + Date dateFinished = p.getDateFinished(); + assertNull(dateFinished); + + // set cron scheduler, once a second + Map<String, Object> config = note.getConfig(); + config.put("cron", "* * * * * ?"); + note.setConfig(config); + notebook.refreshCron(note.id()); + Thread.sleep(1*1000); + dateFinished = p.getDateFinished(); + assertNotNull(dateFinished); + + // remove cron scheduler. + config.put("cron", null); + note.setConfig(config); + notebook.refreshCron(note.id()); + Thread.sleep(1*1000); + assertEquals(dateFinished, p.getDateFinished()); + } + + private void delete(File file){ + if(file.isFile()) file.delete(); + else if(file.isDirectory()){ + File [] files = file.listFiles(); + if(files!=null && files.length>0){ + for(File f : files){ + delete(f); + } + } + file.delete(); + } + } + + @Override + public JobListener getParagraphJobListener(Note note) { + return new JobListener(){ + + @Override + public void onProgressUpdate(Job job, int progress) { + } + + @Override + public void beforeStatusChange(Job job, Status before, Status after) { + } + + @Override + public void afterStatusChange(Job job, Status before, Status after) { + } + }; + } +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilTest.java new file mode 100644 index 0000000..e96c824 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.util; + +import org.apache.zeppelin.util.Util; + +import junit.framework.TestCase; + +public class UtilTest extends TestCase { + + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + } + + public void testSplitIncludingToken() { + String[] token = Util.split("hello | \"world '>|hehe\" > next >> sink", new String[]{"|", ">>", ">"}, true); + assertEquals(7, token.length); + assertEquals(" \"world '>|hehe\" ", token[2]); + } + + public void testSplitExcludingToken() { + String[] token = Util.split("hello | \"world '>|hehe\" > next >> sink", new String[]{"|", ">>", ">"}, false); + assertEquals(4, token.length); + assertEquals(" \"world '>|hehe\" ", token[1]); + } + + public void testSplitWithSemicolonEnd(){ + String[] token = Util.split("show tables;", ';'); + assertEquals(1, token.length); + assertEquals("show tables", token[0]); + } + + public void testEscapeTemplate(){ + String[] token = Util.split("select * from <%=table%> limit 1 > output", '>'); + assertEquals(2, token.length); + assertEquals("output", token[1]); + } + + public void testSplit(){ + String [] op = new String[]{";", "|", ">>", ">"}; + + String str = "CREATE external table news20b_train (\n"+ + " rowid int,\n"+ + " label int,\n"+ + " features ARRAY<STRING>\n"+ + ")\n"+ + "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' \n"+ + "COLLECTION ITEMS TERMINATED BY \",\" \n"+ + "STORED AS TEXTFILE;\n"; + Util.split(str, op, true); + + } + + public void testSplitDifferentBlockStartEnd(){ + String [] op = new String[]{";", "|", ">>", ">"}; + String escapeSeq = "\"',;<%>!"; + char escapeChar = '\\'; + String [] blockStart = new String[]{ "\"", "'", "<%", "<", "!"}; + String [] blockEnd = new String[]{ "\"", "'", "%>", ">", ";" }; + String [] t = Util.split("!echo a;!echo b;", escapeSeq, escapeChar, blockStart, blockEnd, op, true); + assertEquals(4, t.length); + assertEquals("!echo a;", t[0]); + assertEquals(";", t[1]); + assertEquals("!echo b;", t[2]); + assertEquals(";", t[3]); + } + + public void testNestedBlock(){ + String [] op = new String[]{";", "|", ">>", ">"}; + String escapeSeq = "\"',;<%>!"; + char escapeChar = '\\'; + String [] blockStart = new String[]{ "\"", "'", "<%", "N_<", "<", "!"}; + String [] blockEnd = new String[]{ "\"", "'", "%>", "N_>", ";", ";" }; + String [] t = Util.split("array <STRUCT<STRING>> tags|aa", escapeSeq, escapeChar, blockStart, blockEnd, op, true); + assertEquals(3, t.length); + assertEquals("array <STRUCT<STRING>> tags", t[0]); + assertEquals("aa", t[2]); + } +} http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/669d408d/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilsForTests.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilsForTests.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilsForTests.java new file mode 100644 index 0000000..7700d19 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/util/UtilsForTests.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.util; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class UtilsForTests { + + public static File createTmpDir() throws Exception { + File tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); + tmpDir.mkdir(); + return tmpDir; + + } + /* + private static final String HADOOP_DIST="http://apache.mirror.cdnetworks.com/hadoop/common/hadoop-1.2.1/hadoop-1.2.1-bin.tar.gz";; + //private static final String HADOOP_DIST="http://www.us.apache.org/dist/hadoop/common/hadoop-1.2.1/hadoop-1.2.1-bin.tar.gz";; + + public static void getHadoop() throws MalformedURLException, IOException{ + setEnv("HADOOP_HOME", new File("./target/hadoop-1.2.1").getAbsolutePath()); + if(new File("./target/hadoop-1.2.1").isDirectory()) return; + //System.out.println("Downloading a hadoop distribution ... it will take a while"); + //FileUtils.copyURLToFile(new URL(HADOOP_DIST), new File("/tmp/zp_test_hadoop-bin.tar.gz")); + System.out.println("Unarchive hadoop distribution ... "); + new File("./target").mkdir(); + Runtime.getRuntime().exec("tar -xzf /tmp/zp_test_hadoop-bin.tar.gz -C ./target"); + } + */ + + public static void delete(File file){ + if(file.isFile()) file.delete(); + else if(file.isDirectory()){ + File [] files = file.listFiles(); + if(files!=null && files.length>0){ + for(File f : files){ + delete(f); + } + } + file.delete(); + } + } + + /** + * Utility method to create a file (if does not exist) and populate it the the given content + * + * @param path to file + * @param content of the file + * @throws IOException + */ + public static void createFileWithContent(String path, String content) throws IOException { + File f = new File(path); + if (!f.exists()) { + stringToFile(content, f); + } + } + + public static void stringToFile(String string, File file) throws IOException{ + FileOutputStream out = new FileOutputStream(file); + out.write(string.getBytes()); + out.close(); + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public static void setEnv(String k, String v) { + Map<String, String> newenv = new HashMap<String, String>(); + newenv.put(k, v); + try { + Class<?> processEnvironmentClass = Class.forName("java.lang.ProcessEnvironment"); + Field theEnvironmentField = processEnvironmentClass.getDeclaredField("theEnvironment"); + theEnvironmentField.setAccessible(true); + Map<String, String> env = (Map<String, String>) theEnvironmentField.get(null); + env.putAll(newenv); + Field theCaseInsensitiveEnvironmentField = processEnvironmentClass.getDeclaredField("theCaseInsensitiveEnvironment"); + theCaseInsensitiveEnvironmentField.setAccessible(true); + Map<String, String> cienv = (Map<String, String>) theCaseInsensitiveEnvironmentField.get(null); + cienv.putAll(newenv); + } catch (NoSuchFieldException e) { + try { + Class[] classes = Collections.class.getDeclaredClasses(); + Map<String, String> env = System.getenv(); + for(Class cl : classes) { + if("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { + Field field = cl.getDeclaredField("m"); + field.setAccessible(true); + Object obj = field.get(env); + Map<String, String> map = (Map<String, String>) obj; + map.clear(); + map.putAll(newenv); + } + } + } catch (Exception e2) { + e2.printStackTrace(); + } + } catch (Exception e1) { + e1.printStackTrace(); + } + } +}
