http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java ---------------------------------------------------------------------- diff --git a/neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java b/neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java index e398568..24bd513 100644 --- a/neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java +++ b/neo4j/src/test/java/org/apache/zeppelin/graph/neo4j/Neo4jCypherInterpreterTest.java @@ -16,12 +16,7 @@ */ package org.apache.zeppelin.graph.neo4j; -import static org.junit.Assert.assertEquals; - import com.google.gson.Gson; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.graph.neo4j.Neo4jConnectionManager.Neo4jAuthType; import org.apache.zeppelin.interpreter.InterpreterContext; @@ -39,6 +34,12 @@ import org.junit.runners.MethodSorters; import org.neo4j.harness.ServerControls; import org.neo4j.harness.TestServerBuilders; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class Neo4jCypherInterpreterTest { @@ -54,27 +55,25 @@ public class Neo4jCypherInterpreterTest { private static final String REL_KNOWS = "KNOWS"; private static final String CYPHER_FOREACH = - "FOREACH (x in range(1,1000) | CREATE (:%s{name: \"name\" + x, age: %s}))"; - private static final String CHPHER_UNWIND = - "UNWIND range(1,1000) as x " - + "MATCH (n), (m) WHERE id(n) = x AND id(m) = toInt(rand() * 1000) " - + "CREATE (n)-[:%s]->(m)"; + "FOREACH (x in range(1,1000) | CREATE (:%s{name: \"name\" + x, age: %s}))"; + private static final String CHPHER_UNWIND = "UNWIND range(1,1000) as x " + + "MATCH (n), (m) WHERE id(n) = x AND id(m) = toInt(rand() * 1000) " + + "CREATE (n)-[:%s]->(m)"; @BeforeClass public static void setUpNeo4jServer() throws Exception { - server = - TestServerBuilders.newInProcessBuilder() - .withConfig("dbms.security.auth_enabled", "false") - .withFixture(String.format(CYPHER_FOREACH, LABEL_PERSON, "x % 10")) - .withFixture(String.format(CHPHER_UNWIND, REL_KNOWS)) - .newServer(); + server = TestServerBuilders.newInProcessBuilder() + .withConfig("dbms.security.auth_enabled", "false") + .withFixture(String.format(CYPHER_FOREACH, LABEL_PERSON, "x % 10")) + .withFixture(String.format(CHPHER_UNWIND, REL_KNOWS)) + .newServer(); } @AfterClass public static void tearDownNeo4jServer() throws Exception { server.close(); } - + @Before public void setUpZeppelin() { Properties p = new Properties(); @@ -82,8 +81,9 @@ public class Neo4jCypherInterpreterTest { p.setProperty(Neo4jConnectionManager.NEO4J_AUTH_TYPE, Neo4jAuthType.NONE.toString()); p.setProperty(Neo4jConnectionManager.NEO4J_MAX_CONCURRENCY, "50"); interpreter = new Neo4jCypherInterpreter(p); - context = InterpreterContext.builder().setInterpreterOut(new InterpreterOutput(null)).build(); - ; + context = InterpreterContext.builder() + .setInterpreterOut(new InterpreterOutput(null)) + .build();; } @After @@ -94,18 +94,17 @@ public class Neo4jCypherInterpreterTest { @Test public void testTableWithArray() { interpreter.open(); - InterpreterResult result = - interpreter.interpret("return 'a' as colA, 'b' as colB, [1, 2, 3] as colC", context); + InterpreterResult result = interpreter.interpret( + "return 'a' as colA, 'b' as colB, [1, 2, 3] as colC", context); assertEquals(Code.SUCCESS, result.code()); final String tableResult = "colA\tcolB\tcolC\n\"a\"\t\"b\"\t[1,2,3]\n"; assertEquals(tableResult, result.toString().replace("%table ", StringUtils.EMPTY)); - - result = - interpreter.interpret( + + result = interpreter.interpret( "return 'a' as colA, 'b' as colB, [{key: \"value\"}, {key: 1}] as colC", context); assertEquals(Code.SUCCESS, result.code()); final String tableResultWithMap = - "colA\tcolB\tcolC\n\"a\"\t\"b\"\t[{\"key\":\"value\"},{\"key\":1}]\n"; + "colA\tcolB\tcolC\n\"a\"\t\"b\"\t[{\"key\":\"value\"},{\"key\":1}]\n"; assertEquals(tableResultWithMap, result.toString().replace("%table ", StringUtils.EMPTY)); } @@ -120,12 +119,9 @@ public class Neo4jCypherInterpreterTest { @Test public void testRenderTable() { interpreter.open(); - InterpreterResult result = - interpreter.interpret( - "MATCH (n:Person) " - + "WHERE n.name IN ['name1', 'name2', 'name3'] " - + "RETURN n.name AS name, n.age AS age", - context); + InterpreterResult result = interpreter.interpret("MATCH (n:Person) " + + "WHERE n.name IN ['name1', 'name2', 'name3'] " + + "RETURN n.name AS name, n.age AS age", context); assertEquals(Code.SUCCESS, result.code()); final String tableResult = "name\tage\n\"name1\"\t1\n\"name2\"\t2\n\"name3\"\t3\n"; assertEquals(tableResult, result.toString().replace("%table ", StringUtils.EMPTY)); @@ -135,15 +131,12 @@ public class Neo4jCypherInterpreterTest { public void testRenderMap() { interpreter.open(); final String jsonQuery = - "RETURN {key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]} as object"; + "RETURN {key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]} as object"; final String objectKey = "object.key"; final String objectListKey = "object.listKey"; InterpreterResult result = interpreter.interpret(jsonQuery, context); assertEquals(Code.SUCCESS, result.code()); - String[] rows = - result - .toString() - .replace("%table ", StringUtils.EMPTY) + String[] rows = result.toString().replace("%table ", StringUtils.EMPTY) .split(Neo4jCypherInterpreter.NEW_LINE); assertEquals(rows.length, 2); List<String> header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB)); @@ -152,19 +145,15 @@ public class Neo4jCypherInterpreterTest { List<String> row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB)); assertEquals(row.size(), header.size()); assertEquals(row.get(header.indexOf(objectKey)), "value"); - assertEquals( - row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]"); + assertEquals(row.get(header.indexOf(objectListKey)), + "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]"); - final String query = - "WITH [{key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]}," + final String query = "WITH [{key: \"value\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]}," + "{key: \"value2\", listKey: [{inner: \"Map12\"}, {inner: \"Map22\"}]}] " + "AS array UNWIND array AS object RETURN object"; result = interpreter.interpret(query, context); assertEquals(Code.SUCCESS, result.code()); - rows = - result - .toString() - .replace("%table ", StringUtils.EMPTY) + rows = result.toString().replace("%table ", StringUtils.EMPTY) .split(Neo4jCypherInterpreter.NEW_LINE); assertEquals(rows.length, 3); header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB)); @@ -173,24 +162,20 @@ public class Neo4jCypherInterpreterTest { row = Arrays.asList(rows[1].split(Neo4jCypherInterpreter.TAB)); assertEquals(row.size(), header.size()); assertEquals(row.get(header.indexOf(objectKey)), "value"); - assertEquals( - row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]"); + assertEquals(row.get(header.indexOf(objectListKey)), + "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]"); row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB)); assertEquals(row.size(), header.size()); assertEquals(row.get(header.indexOf(objectKey)), "value2"); - assertEquals( - row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map12\"},{\"inner\":\"Map22\"}]"); + assertEquals(row.get(header.indexOf(objectListKey)), + "[{\"inner\":\"Map12\"},{\"inner\":\"Map22\"}]"); - final String jsonListWithNullQuery = - "WITH [{key: \"value\", listKey: null}," + final String jsonListWithNullQuery = "WITH [{key: \"value\", listKey: null}," + "{key: \"value2\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]}] " + "AS array UNWIND array AS object RETURN object"; result = interpreter.interpret(jsonListWithNullQuery, context); assertEquals(Code.SUCCESS, result.code()); - rows = - result - .toString() - .replace("%table ", StringUtils.EMPTY) + rows = result.toString().replace("%table ", StringUtils.EMPTY) .split(Neo4jCypherInterpreter.NEW_LINE); assertEquals(rows.length, 3); header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB, -1)); @@ -204,19 +189,15 @@ public class Neo4jCypherInterpreterTest { row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB, -1)); assertEquals(row.size(), header.size()); assertEquals(row.get(header.indexOf(objectKey)), "value2"); - assertEquals( - row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]"); - - final String jsonListWithoutListKeyQuery = - "WITH [{key: \"value\"}," + assertEquals(row.get(header.indexOf(objectListKey)), + "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]"); + + final String jsonListWithoutListKeyQuery = "WITH [{key: \"value\"}," + "{key: \"value2\", listKey: [{inner: \"Map1\"}, {inner: \"Map2\"}]}] " + "AS array UNWIND array AS object RETURN object"; result = interpreter.interpret(jsonListWithoutListKeyQuery, context); assertEquals(Code.SUCCESS, result.code()); - rows = - result - .toString() - .replace("%table ", StringUtils.EMPTY) + rows = result.toString().replace("%table ", StringUtils.EMPTY) .split(Neo4jCypherInterpreter.NEW_LINE); assertEquals(rows.length, 3); header = Arrays.asList(rows[0].split(Neo4jCypherInterpreter.TAB, -1)); @@ -229,18 +210,17 @@ public class Neo4jCypherInterpreterTest { row = Arrays.asList(rows[2].split(Neo4jCypherInterpreter.TAB, -1)); assertEquals(row.size(), header.size()); assertEquals(row.get(header.indexOf(objectKey)), "value2"); - assertEquals( - row.get(header.indexOf(objectListKey)), "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]"); + assertEquals(row.get(header.indexOf(objectListKey)), + "[{\"inner\":\"Map1\"},{\"inner\":\"Map2\"}]"); } @Test public void testRenderNetwork() { interpreter.open(); - InterpreterResult result = - interpreter.interpret("MATCH (n)-[r:KNOWS]-(m) RETURN n, r, m LIMIT 1", context); - GraphResult.Graph graph = - gson.fromJson( - result.toString().replace("%network ", StringUtils.EMPTY), GraphResult.Graph.class); + InterpreterResult result = interpreter.interpret( + "MATCH (n)-[r:KNOWS]-(m) RETURN n, r, m LIMIT 1", context); + GraphResult.Graph graph = gson.fromJson(result.toString().replace("%network ", + StringUtils.EMPTY), GraphResult.Graph.class); assertEquals(2, graph.getNodes().size()); assertEquals(true, graph.getNodes().iterator().next().getLabel().equals(LABEL_PERSON)); assertEquals(1, graph.getEdges().size()); @@ -264,9 +244,8 @@ public class Neo4jCypherInterpreterTest { assertEquals(Code.SUCCESS, result.code()); assertEquals(errorMsgEmpty, result.toString()); - result = - interpreter.interpret( - "MATCH (n:Person{name: }) RETURN n.name AS name, n.age AS age", context); + result = interpreter.interpret("MATCH (n:Person{name: }) RETURN n.name AS name, n.age AS age", + context); assertEquals(Code.ERROR, result.code()); } }
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/pom.xml ---------------------------------------------------------------------- diff --git a/pig/pom.xml b/pig/pom.xml index 4553b5c..571d198 100644 --- a/pig/pom.xml +++ b/pig/pom.xml @@ -190,6 +190,13 @@ <forkMode>always</forkMode> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <configuration> + <skip>false</skip> + </configuration> + </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java ---------------------------------------------------------------------- diff --git a/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java b/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java index 97d15de..9503aa7 100644 --- a/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java +++ b/pig/src/main/java/org/apache/zeppelin/pig/BasePigInterpreter.java @@ -17,24 +17,28 @@ package org.apache.zeppelin.pig; -import java.lang.reflect.Field; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.pig.PigServer; import org.apache.pig.backend.BackendException; import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine; import org.apache.pig.backend.hadoop.executionengine.Launcher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Field; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -/** */ +/** + * + */ public abstract class BasePigInterpreter extends Interpreter { private static final Logger LOGGER = LoggerFactory.getLogger(BasePigInterpreter.class); @@ -56,7 +60,7 @@ public abstract class BasePigInterpreter extends Interpreter { for (String jobId : jobIds) { LOGGER.info("Kill jobId:" + jobId); HExecutionEngine engine = - (HExecutionEngine) getPigServer().getPigContext().getExecutionEngine(); + (HExecutionEngine) getPigServer().getPigContext().getExecutionEngine(); try { Field launcherField = HExecutionEngine.class.getDeclaredField("launcher"); launcherField.setAccessible(true); @@ -68,8 +72,8 @@ public abstract class BasePigInterpreter extends Interpreter { } } } else { - LOGGER.warn( - "No PigScriptListener found, can not cancel paragraph:" + context.getParagraphId()); + LOGGER.warn("No PigScriptListener found, can not cancel paragraph:" + + context.getParagraphId()); } } @@ -89,15 +93,14 @@ public abstract class BasePigInterpreter extends Interpreter { @Override public Scheduler getScheduler() { - return SchedulerFactory.singleton() - .createOrGetFIFOScheduler(PigInterpreter.class.getName() + this.hashCode()); + return SchedulerFactory.singleton().createOrGetFIFOScheduler( + PigInterpreter.class.getName() + this.hashCode()); } public abstract PigServer getPigServer(); /** * Use paragraph title if it exists, else use the last line of pig script. - * * @param cmd * @param context * @return http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java ---------------------------------------------------------------------- diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java b/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java index 5f79e97..4fc0676 100644 --- a/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java +++ b/pig/src/main/java/org/apache/zeppelin/pig/PigInterpreter.java @@ -17,11 +17,6 @@ package org.apache.zeppelin.pig; -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.util.Map; -import java.util.Properties; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.pig.PigServer; @@ -29,13 +24,22 @@ import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.tools.pigscript.parser.ParseException; import org.apache.pig.tools.pigstats.PigStats; import org.apache.pig.tools.pigstats.ScriptState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Map; +import java.util.Properties; + import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -/** Pig interpreter for Zeppelin. */ +/** + * Pig interpreter for Zeppelin. + */ public class PigInterpreter extends BasePigInterpreter { private static final Logger LOGGER = LoggerFactory.getLogger(PigInterpreter.class); @@ -60,10 +64,8 @@ public class PigInterpreter extends BasePigInterpreter { pigServer = new PigServer(execType); for (Map.Entry entry : getProperties().entrySet()) { if (!entry.getKey().toString().startsWith("zeppelin.")) { - pigServer - .getPigContext() - .getProperties() - .setProperty(entry.getKey().toString(), entry.getValue().toString()); + pigServer.getPigContext().getProperties().setProperty(entry.getKey().toString(), + entry.getValue().toString()); } } } catch (IOException e) { @@ -77,6 +79,7 @@ public class PigInterpreter extends BasePigInterpreter { pigServer = null; } + @Override public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) { // remember the origial stdout, because we will redirect stdout to capture @@ -143,7 +146,10 @@ public class PigInterpreter extends BasePigInterpreter { return new InterpreterResult(Code.SUCCESS, outputBuilder.toString()); } + public PigServer getPigServer() { return pigServer; } + } + http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java ---------------------------------------------------------------------- diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java b/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java index 0c6eef6..97c2f6e 100644 --- a/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java +++ b/pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java @@ -17,12 +17,6 @@ package org.apache.zeppelin.pig; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Properties; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.pig.PigServer; @@ -32,15 +26,25 @@ import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.tools.pigscript.parser.ParseException; import org.apache.pig.tools.pigstats.PigStats; import org.apache.pig.tools.pigstats.ScriptState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; + import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.ResultMessages; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -/** */ +/** + * + */ public class PigQueryInterpreter extends BasePigInterpreter { private static final Logger LOGGER = LoggerFactory.getLogger(PigQueryInterpreter.class); private static final String MAX_RESULTS = "zeppelin.pig.maxResult"; @@ -58,7 +62,9 @@ public class PigQueryInterpreter extends BasePigInterpreter { } @Override - public void close() {} + public void close() { + + } @Override public InterpreterResult interpret(String st, InterpreterContext context) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java ---------------------------------------------------------------------- diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java b/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java index 0e6068a..8ff1bf8 100644 --- a/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java +++ b/pig/src/main/java/org/apache/zeppelin/pig/PigScriptListener.java @@ -17,8 +17,6 @@ package org.apache.zeppelin.pig; -import java.util.HashSet; -import java.util.Set; import org.apache.pig.impl.plan.OperatorPlan; import org.apache.pig.tools.pigstats.JobStats; import org.apache.pig.tools.pigstats.OutputStats; @@ -26,7 +24,12 @@ import org.apache.pig.tools.pigstats.PigProgressNotificationListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** */ +import java.util.HashSet; +import java.util.Set; + +/** + * + */ public class PigScriptListener implements PigProgressNotificationListener { private static final Logger LOGGER = LoggerFactory.getLogger(PigScriptListener.class); @@ -34,13 +37,19 @@ public class PigScriptListener implements PigProgressNotificationListener { private int progress; @Override - public void initialPlanNotification(String scriptId, OperatorPlan<?> plan) {} + public void initialPlanNotification(String scriptId, OperatorPlan<?> plan) { + + } @Override - public void launchStartedNotification(String scriptId, int numJobsToLaunch) {} + public void launchStartedNotification(String scriptId, int numJobsToLaunch) { + + } @Override - public void jobsSubmittedNotification(String scriptId, int numJobsSubmitted) {} + public void jobsSubmittedNotification(String scriptId, int numJobsSubmitted) { + + } @Override public void jobStartedNotification(String scriptId, String assignedJobId) { @@ -48,13 +57,19 @@ public class PigScriptListener implements PigProgressNotificationListener { } @Override - public void jobFinishedNotification(String scriptId, JobStats jobStats) {} + public void jobFinishedNotification(String scriptId, JobStats jobStats) { + + } @Override - public void jobFailedNotification(String scriptId, JobStats jobStats) {} + public void jobFailedNotification(String scriptId, JobStats jobStats) { + + } @Override - public void outputCompletedNotification(String scriptId, OutputStats outputStats) {} + public void outputCompletedNotification(String scriptId, OutputStats outputStats) { + + } @Override public void progressUpdatedNotification(String scriptId, int progress) { @@ -63,7 +78,9 @@ public class PigScriptListener implements PigProgressNotificationListener { } @Override - public void launchCompletedNotification(String scriptId, int numJobsSucceeded) {} + public void launchCompletedNotification(String scriptId, int numJobsSucceeded) { + + } public Set<String> getJobIds() { return jobIds; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java ---------------------------------------------------------------------- diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java b/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java index 28fd763..1c48250 100644 --- a/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java +++ b/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java @@ -17,16 +17,19 @@ package org.apache.zeppelin.pig; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.List; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** */ +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.List; + +/** + * + */ public class PigUtils { private static final Logger LOGGER = LoggerFactory.getLogger(PigUtils.class); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java ---------------------------------------------------------------------- diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java index 1fd5c82..ea1a3f8 100644 --- a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java +++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterSparkTest.java @@ -1,31 +1,37 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * <p>http://www.apache.org/licenses/LICENSE-2.0 - * - * <p>Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.zeppelin.pig; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import org.apache.commons.io.IOUtils; +import org.junit.After; +import org.junit.Test; + import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.Properties; -import org.apache.commons.io.IOUtils; + import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; -import org.junit.After; -import org.junit.Test; public class PigInterpreterSparkTest { private PigInterpreter pigInterpreter; @@ -38,8 +44,8 @@ public class PigInterpreterSparkTest { pigInterpreter = new PigInterpreter(properties); pigInterpreter.open(); context = InterpreterContext.builder().setParagraphId("paragraphId").build(); - } + } @After public void tearDown() { pigInterpreter.close(); @@ -49,44 +55,41 @@ public class PigInterpreterSparkTest { public void testBasics() throws IOException { setUpSpark(false); - String content = "1\tandy\n" + "2\tpeter\n"; + String content = "1\tandy\n" + + "2\tpeter\n"; File tmpFile = File.createTempFile("zeppelin", "test"); FileWriter writer = new FileWriter(tmpFile); IOUtils.write(content, writer); writer.close(); // simple pig script using dump - String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "dump a;"; + String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + + "dump a;"; InterpreterResult result = pigInterpreter.interpret(pigscript, context); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)")); // describe - pigscript = - "a = load '" - + tmpFile.getAbsolutePath() - + "' as (id: int, name: bytearray);" - + "describe a;"; + pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);" + + "describe a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}")); // syntax error (compilation error) - pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + "describe a;"; + pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + + "describe a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.ERROR, result.code()); - assertTrue( - result - .message() - .get(0) - .getData() - .contains("Syntax error, unexpected symbol at or near 'a'")); + assertTrue(result.message().get(0).getData().contains( + "Syntax error, unexpected symbol at or near 'a'")); // syntax error - pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "foreach a generate $0;"; + pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + + "foreach a generate $0;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.ERROR, result.code()); @@ -97,14 +100,16 @@ public class PigInterpreterSparkTest { public void testIncludeJobStats() throws IOException { setUpSpark(true); - String content = "1\tandy\n" + "2\tpeter\n"; + String content = "1\tandy\n" + + "2\tpeter\n"; File tmpFile = File.createTempFile("zeppelin", "test"); FileWriter writer = new FileWriter(tmpFile); IOUtils.write(content, writer); writer.close(); // simple pig script using dump - String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "dump a;"; + String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + + "dump a;"; InterpreterResult result = pigInterpreter.interpret(pigscript, context); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); @@ -112,11 +117,8 @@ public class PigInterpreterSparkTest { assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)")); // describe - pigscript = - "a = load '" - + tmpFile.getAbsolutePath() - + "' as (id: int, name: bytearray);" - + "describe a;"; + pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);" + + "describe a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); @@ -124,23 +126,24 @@ public class PigInterpreterSparkTest { assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}")); // syntax error (compilation error) - pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + "describe a;"; + pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + + "describe a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.ERROR, result.code()); // no job is launched, so no jobStats - assertTrue( - result - .message() - .get(0) - .getData() - .contains("Syntax error, unexpected symbol at or near 'a'")); + assertTrue(result.message().get(0).getData().contains( + "Syntax error, unexpected symbol at or near 'a'")); // execution error - pigscript = "a = load 'invalid_path';" + "dump a;"; + pigscript = "a = load 'invalid_path';" + + "dump a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.ERROR, result.code()); assertTrue(result.message().get(0).getData().contains("Failed to read data from")); } + } + + http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java ---------------------------------------------------------------------- diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java index f1f6ad8..5a21bb3 100644 --- a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java +++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java @@ -1,33 +1,39 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * <p>http://www.apache.org/licenses/LICENSE-2.0 - * - * <p>Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.zeppelin.pig; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import org.apache.commons.io.IOUtils; +import org.junit.After; +import org.junit.Test; + import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.Properties; -import org.apache.commons.io.IOUtils; + import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterResult.Type; -import org.junit.After; -import org.junit.Test; public class PigInterpreterTest { @@ -52,24 +58,23 @@ public class PigInterpreterTest { public void testBasics() throws IOException { setUpLocal(false); - String content = "1\tandy\n" + "2\tpeter\n"; + String content = "1\tandy\n" + + "2\tpeter\n"; File tmpFile = File.createTempFile("zeppelin", "test"); FileWriter writer = new FileWriter(tmpFile); IOUtils.write(content, writer); writer.close(); // simple pig script using dump - String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "dump a;"; + String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + + "dump a;"; InterpreterResult result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.SUCCESS, result.code()); assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)")); // describe - pigscript = - "a = load '" - + tmpFile.getAbsolutePath() - + "' as (id: int, name: bytearray);" + pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);" + "describe a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); @@ -77,37 +82,38 @@ public class PigInterpreterTest { assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}")); // syntax error (compilation error) - pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + "describe a;"; + pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + + "describe a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.ERROR, result.code()); - assertTrue( - result - .message() - .get(0) - .getData() - .contains("Syntax error, unexpected symbol at or near 'a'")); + assertTrue(result.message().get(0).getData().contains( + "Syntax error, unexpected symbol at or near 'a'")); // execution error - pigscript = "a = load 'invalid_path';" + "dump a;"; + pigscript = "a = load 'invalid_path';" + + "dump a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.ERROR, result.code()); assertTrue(result.message().get(0).getData().contains("Input path does not exist")); } + @Test public void testIncludeJobStats() throws IOException { setUpLocal(true); - String content = "1\tandy\n" + "2\tpeter\n"; + String content = "1\tandy\n" + + "2\tpeter\n"; File tmpFile = File.createTempFile("zeppelin", "test"); FileWriter writer = new FileWriter(tmpFile); IOUtils.write(content, writer); writer.close(); // simple pig script using dump - String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "dump a;"; + String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + + "dump a;"; InterpreterResult result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.SUCCESS, result.code()); @@ -115,10 +121,7 @@ public class PigInterpreterTest { assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)")); // describe - pigscript = - "a = load '" - + tmpFile.getAbsolutePath() - + "' as (id: int, name: bytearray);" + pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);" + "describe a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); @@ -128,25 +131,24 @@ public class PigInterpreterTest { assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}")); // syntax error (compilation error) - pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + "describe a;"; + pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + + "describe a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.ERROR, result.code()); // no job is launched, so no jobStats assertTrue(!result.message().get(0).getData().contains("Counters:")); - assertTrue( - result - .message() - .get(0) - .getData() - .contains("Syntax error, unexpected symbol at or near 'a'")); + assertTrue(result.message().get(0).getData().contains( + "Syntax error, unexpected symbol at or near 'a'")); // execution error - pigscript = "a = load 'invalid_path';" + "dump a;"; + pigscript = "a = load 'invalid_path';" + + "dump a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.ERROR, result.code()); assertTrue(result.message().get(0).getData().contains("Counters:")); assertTrue(result.message().get(0).getData().contains("Input path does not exist")); } + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java ---------------------------------------------------------------------- diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java index 14d9686..ec09a88 100644 --- a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java +++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java @@ -1,33 +1,39 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * <p>http://www.apache.org/licenses/LICENSE-2.0 - * - * <p>Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.zeppelin.pig; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import org.apache.commons.io.IOUtils; +import org.junit.After; +import org.junit.Test; + import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.Properties; -import org.apache.commons.io.IOUtils; + import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterResult.Type; -import org.junit.After; -import org.junit.Test; public class PigInterpreterTezTest { @@ -42,8 +48,8 @@ public class PigInterpreterTezTest { pigInterpreter = new PigInterpreter(properties); pigInterpreter.open(); context = InterpreterContext.builder().setParagraphId("paragraphId").build(); - } + } @After public void tearDown() { pigInterpreter.close(); @@ -53,52 +59,45 @@ public class PigInterpreterTezTest { public void testBasics() throws IOException { setUpTez(false); - assertEquals( - "test", - pigInterpreter - .getPigServer() - .getPigContext() - .getProperties() + assertEquals("test", + pigInterpreter.getPigServer().getPigContext().getProperties() .getProperty("tez.queue.name")); - - String content = "1\tandy\n" + "2\tpeter\n"; + + String content = "1\tandy\n" + + "2\tpeter\n"; File tmpFile = File.createTempFile("zeppelin", "test"); FileWriter writer = new FileWriter(tmpFile); IOUtils.write(content, writer); writer.close(); // simple pig script using dump - String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "dump a;"; + String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + + "dump a;"; InterpreterResult result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.SUCCESS, result.code()); assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)")); // describe - pigscript = - "a = load '" - + tmpFile.getAbsolutePath() - + "' as (id: int, name: bytearray);" - + "describe a;"; + pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);" + + "describe a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.SUCCESS, result.code()); assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}")); // syntax error (compilation error) - pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + "describe a;"; + pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + + "describe a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.ERROR, result.code()); - assertTrue( - result - .message() - .get(0) - .getData() - .contains("Syntax error, unexpected symbol at or near 'a'")); + assertTrue(result.message().get(0).getData().contains( + "Syntax error, unexpected symbol at or near 'a'")); // syntax error - pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "foreach a generate $0;"; + pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + + "foreach a generate $0;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.ERROR, result.code()); @@ -109,14 +108,16 @@ public class PigInterpreterTezTest { public void testIncludeJobStats() throws IOException { setUpTez(true); - String content = "1\tandy\n" + "2\tpeter\n"; + String content = "1\tandy\n" + + "2\tpeter\n"; File tmpFile = File.createTempFile("zeppelin", "test"); FileWriter writer = new FileWriter(tmpFile); IOUtils.write(content, writer); writer.close(); // simple pig script using dump - String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + "dump a;"; + String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';" + + "dump a;"; InterpreterResult result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.SUCCESS, result.code()); @@ -124,11 +125,8 @@ public class PigInterpreterTezTest { assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)")); // describe - pigscript = - "a = load '" - + tmpFile.getAbsolutePath() - + "' as (id: int, name: bytearray);" - + "describe a;"; + pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);" + + "describe a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.SUCCESS, result.code()); @@ -137,21 +135,19 @@ public class PigInterpreterTezTest { assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}")); // syntax error (compilation error) - pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + "describe a;"; + pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';" + + "describe a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.ERROR, result.code()); // no job is launched, so no jobStats assertTrue(!result.message().get(0).getData().contains("Vertex Stats")); - assertTrue( - result - .message() - .get(0) - .getData() - .contains("Syntax error, unexpected symbol at or near 'a'")); + assertTrue(result.message().get(0).getData().contains( + "Syntax error, unexpected symbol at or near 'a'")); // execution error - pigscript = "a = load 'invalid_path';" + "dump a;"; + pigscript = "a = load 'invalid_path';" + + "dump a;"; result = pigInterpreter.interpret(pigscript, context); assertEquals(Type.TEXT, result.message().get(0).getType()); assertEquals(Code.ERROR, result.code()); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java ---------------------------------------------------------------------- diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java index a850174..01f0a20 100644 --- a/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java +++ b/pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java @@ -1,40 +1,48 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * <p>http://www.apache.org/licenses/LICENSE-2.0 - * - * <p>Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.zeppelin.pig; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import org.apache.commons.io.IOUtils; +import org.apache.zeppelin.interpreter.LazyOpenInterpreter; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Properties; -import org.apache.commons.io.IOUtils; + import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.LazyOpenInterpreter; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -/** */ +/** + * + */ public class PigQueryInterpreterTest { private Interpreter pigInterpreter; @@ -70,28 +78,23 @@ public class PigQueryInterpreterTest { @Test public void testBasics() throws IOException, InterpreterException { - String content = "andy\tmale\t10\n" + "peter\tmale\t20\n" + "amy\tfemale\t14\n"; + String content = "andy\tmale\t10\n" + + "peter\tmale\t20\n" + + "amy\tfemale\t14\n"; File tmpFile = File.createTempFile("zeppelin", "test"); FileWriter writer = new FileWriter(tmpFile); IOUtils.write(content, writer); writer.close(); // run script in PigInterpreter - String pigscript = - "a = load '" - + tmpFile.getAbsolutePath() - + "' as (name, gender, age);\n" + String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (name, gender, age);\n" + "a2 = load 'invalid_path' as (name, gender, age);\n" + "dump a;"; InterpreterResult result = pigInterpreter.interpret(pigscript, context); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); - assertTrue( - result - .message() - .get(0) - .getData() - .contains("(andy,male,10)\n(peter,male,20)\n(amy,female,14)")); + assertTrue(result.message().get(0).getData().contains( + "(andy,male,10)\n(peter,male,20)\n(amy,female,14)")); // run single line query in PigQueryInterpreter String query = "foreach a generate name, age;"; @@ -115,18 +118,13 @@ public class PigQueryInterpreterTest { assertEquals("group\tcol_1\nmale\t2\nfemale\t1\n", result.message().get(0).getData()); // syntax error in PigQueryInterpereter - query = - "b = group a by invalid_column;\nforeach b generate group as gender, " - + "COUNT($1) as count;"; + query = "b = group a by invalid_column;\nforeach b generate group as gender, " + + "COUNT($1) as count;"; result = pigQueryInterpreter.interpret(query, context); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertEquals(InterpreterResult.Code.ERROR, result.code()); - assertTrue( - result - .message() - .get(0) - .getData() - .contains("Projected field [invalid_column] does not exist in schema")); + assertTrue(result.message().get(0).getData().contains( + "Projected field [invalid_column] does not exist in schema")); // execution error in PigQueryInterpreter query = "foreach a2 generate name, age;"; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index a9d3cc0..60af8bc 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,6 @@ <plugin.download.version>1.3.0</plugin.download.version> <plugin.deploy.version>2.8.2</plugin.deploy.version> <plugin.shade.version>3.1.1</plugin.shade.version> - <plugin.fmt.version>2.5.1</plugin.fmt.version> <PermGen>64m</PermGen> <MaxPermGen>512m</MaxPermGen> @@ -398,6 +397,46 @@ </plugin> <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <version>2.17</version> + <configuration> + <skip>true</skip> + <failOnViolation>false</failOnViolation> + <includeTestSourceDirectory>true</includeTestSourceDirectory> + <sourceDirectories>${basedir}/src/main/java,${basedir}/src/main/scala</sourceDirectories> + <testSourceDirectory>${basedir}/src/test/java</testSourceDirectory> + <configLocation>_tools/checkstyle.xml</configLocation> + <outputFile>${basedir}/target/checkstyle-output.xml</outputFile> + <inputEncoding>${project.build.sourceEncoding}</inputEncoding> + <outputEncoding>${project.reporting.outputEncoding}</outputEncoding> + </configuration> + <executions> + <execution> + <id>checkstyle-fail-build</id> + <phase>validate</phase> + <goals> + <goal>check</goal> + </goals> + <configuration> + <failOnViolation>true</failOnViolation> + <excludes>org/apache/zeppelin/interpreter/thrift/*,org/apache/zeppelin/scio/avro/*,org/apache/zeppelin/python/proto/*</excludes> + </configuration> + </execution> + <execution> + <id>checkstyle-gen-html-report</id> + <phase>install</phase> + <goals> + <goal>checkstyle-aggregate</goal> + </goals> + <configuration> + <excludes>org/apache/zeppelin/interpreter/thrift/*,org/apache/zeppelin/scio/avro/*,org/apache/zeppelin/python/proto/*</excludes> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> <artifactId>maven-resources-plugin</artifactId> <version>${plugin.resource.version}</version> <executions> @@ -530,29 +569,14 @@ </configuration> </plugin> --> - <plugin> - <groupId>com.coveo</groupId> - <artifactId>fmt-maven-plugin</artifactId> - </plugin> </plugins> <pluginManagement> <plugins> <plugin> - <groupId>com.coveo</groupId> - <artifactId>fmt-maven-plugin</artifactId> - <version>${plugin.fmt.version}</version> - <configuration> - <displayLimit>10000</displayLimit> - </configuration> - <executions> - <execution> - <phase>validate</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <version>${plugin.checkstyle.version}</version> </plugin> <plugin> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/pom.xml ---------------------------------------------------------------------- diff --git a/python/pom.xml b/python/pom.xml index b60e423..d11c165 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -175,6 +175,14 @@ <plugin> <artifactId>maven-resources-plugin</artifactId> </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <configuration> + <skip>false</skip> + </configuration> + </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java b/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java index 200fcf6..b9c897b 100644 --- a/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java +++ b/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java @@ -1,30 +1,25 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ package org.apache.zeppelin.python; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.stub.StreamObserver; -import java.io.IOException; -import java.security.SecureRandom; -import java.util.Iterator; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.zeppelin.interpreter.util.InterpreterOutputStream; import org.apache.zeppelin.python.proto.CancelRequest; @@ -42,7 +37,15 @@ import org.apache.zeppelin.python.proto.StopRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Grpc client for IPython kernel */ +import java.io.IOException; +import java.security.SecureRandom; +import java.util.Iterator; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Grpc client for IPython kernel + */ public class IPythonClient { private static final Logger LOGGER = LoggerFactory.getLogger(IPythonClient.class.getName()); @@ -53,12 +56,16 @@ public class IPythonClient { private SecureRandom random = new SecureRandom(); - /** Construct client for accessing RouteGuide server at {@code host:port}. */ + /** + * Construct client for accessing RouteGuide server at {@code host:port}. + */ public IPythonClient(String host, int port) { this(ManagedChannelBuilder.forAddress(host, port).usePlaintext(true)); } - /** Construct client for accessing RouteGuide server using the existing channel. */ + /** + * Construct client for accessing RouteGuide server using the existing channel. + */ public IPythonClient(ManagedChannelBuilder<?> channelBuilder) { channel = channelBuilder.build(); blockingStub = IPythonGrpc.newBlockingStub(channel); @@ -71,91 +78,86 @@ public class IPythonClient { // execute the code and make the output as streaming by writing it to InterpreterOutputStream // one by one. - public ExecuteResponse stream_execute( - ExecuteRequest request, final InterpreterOutputStream interpreterOutput) { - final ExecuteResponse.Builder finalResponseBuilder = - ExecuteResponse.newBuilder().setStatus(ExecuteStatus.SUCCESS); + public ExecuteResponse stream_execute(ExecuteRequest request, + final InterpreterOutputStream interpreterOutput) { + final ExecuteResponse.Builder finalResponseBuilder = ExecuteResponse.newBuilder() + .setStatus(ExecuteStatus.SUCCESS); final AtomicBoolean completedFlag = new AtomicBoolean(false); LOGGER.debug("stream_execute code:\n" + request.getCode()); - asyncStub.execute( - request, - new StreamObserver<ExecuteResponse>() { - int index = 0; - boolean isPreviousOutputImage = false; - - @Override - public void onNext(ExecuteResponse executeResponse) { - if (executeResponse.getType() == OutputType.TEXT) { - try { - LOGGER.debug("Interpreter Streaming Output: " + executeResponse.getOutput()); - if (isPreviousOutputImage) { - // add '\n' when switch from image to text - interpreterOutput.write("\n%text ".getBytes()); - } - isPreviousOutputImage = false; - interpreterOutput.write(executeResponse.getOutput().getBytes()); - interpreterOutput.getInterpreterOutput().flush(); - } catch (IOException e) { - LOGGER.error("Unexpected IOException", e); - } - } - if (executeResponse.getType() == OutputType.IMAGE) { - try { - LOGGER.debug("Interpreter Streaming Output: IMAGE_DATA"); - if (index != 0) { - // add '\n' if this is the not the first element. otherwise it would mix the image - // with the text - interpreterOutput.write("\n".getBytes()); - } - interpreterOutput.write(("%img " + executeResponse.getOutput()).getBytes()); - interpreterOutput.getInterpreterOutput().flush(); - isPreviousOutputImage = true; - } catch (IOException e) { - LOGGER.error("Unexpected IOException", e); - } + asyncStub.execute(request, new StreamObserver<ExecuteResponse>() { + int index = 0; + boolean isPreviousOutputImage = false; + + @Override + public void onNext(ExecuteResponse executeResponse) { + if (executeResponse.getType() == OutputType.TEXT) { + try { + LOGGER.debug("Interpreter Streaming Output: " + executeResponse.getOutput()); + if (isPreviousOutputImage) { + // add '\n' when switch from image to text + interpreterOutput.write("\n%text ".getBytes()); } - if (executeResponse.getStatus() == ExecuteStatus.ERROR) { - // set the finalResponse to ERROR if any ERROR happens, otherwise the finalResponse - // would - // be SUCCESS. - finalResponseBuilder.setStatus(ExecuteStatus.ERROR); + isPreviousOutputImage = false; + interpreterOutput.write(executeResponse.getOutput().getBytes()); + interpreterOutput.getInterpreterOutput().flush(); + } catch (IOException e) { + LOGGER.error("Unexpected IOException", e); + } + } + if (executeResponse.getType() == OutputType.IMAGE) { + try { + LOGGER.debug("Interpreter Streaming Output: IMAGE_DATA"); + if (index != 0) { + // add '\n' if this is the not the first element. otherwise it would mix the image + // with the text + interpreterOutput.write("\n".getBytes()); } - index++; + interpreterOutput.write(("%img " + executeResponse.getOutput()).getBytes()); + interpreterOutput.getInterpreterOutput().flush(); + isPreviousOutputImage = true; + } catch (IOException e) { + LOGGER.error("Unexpected IOException", e); } + } + if (executeResponse.getStatus() == ExecuteStatus.ERROR) { + // set the finalResponse to ERROR if any ERROR happens, otherwise the finalResponse would + // be SUCCESS. + finalResponseBuilder.setStatus(ExecuteStatus.ERROR); + } + index++; + } - @Override - public void onError(Throwable throwable) { - try { - interpreterOutput - .getInterpreterOutput() - .write(ExceptionUtils.getStackTrace(throwable)); - interpreterOutput.getInterpreterOutput().flush(); - } catch (IOException e) { - LOGGER.error("Unexpected IOException", e); - } - LOGGER.error("Fail to call IPython grpc", throwable); - finalResponseBuilder.setStatus(ExecuteStatus.ERROR); + @Override + public void onError(Throwable throwable) { + try { + interpreterOutput.getInterpreterOutput().write(ExceptionUtils.getStackTrace(throwable)); + interpreterOutput.getInterpreterOutput().flush(); + } catch (IOException e) { + LOGGER.error("Unexpected IOException", e); + } + LOGGER.error("Fail to call IPython grpc", throwable); + finalResponseBuilder.setStatus(ExecuteStatus.ERROR); - completedFlag.set(true); - synchronized (completedFlag) { - completedFlag.notify(); - } - } + completedFlag.set(true); + synchronized (completedFlag) { + completedFlag.notify(); + } + } - @Override - public void onCompleted() { - synchronized (completedFlag) { - try { - LOGGER.debug("stream_execute is completed"); - interpreterOutput.getInterpreterOutput().flush(); - } catch (IOException e) { - LOGGER.error("Unexpected IOException", e); - } - completedFlag.set(true); - completedFlag.notify(); - } + @Override + public void onCompleted() { + synchronized (completedFlag) { + try { + LOGGER.debug("stream_execute is completed"); + interpreterOutput.getInterpreterOutput().flush(); + } catch (IOException e) { + LOGGER.error("Unexpected IOException", e); } - }); + completedFlag.set(true); + completedFlag.notify(); + } + } + }); synchronized (completedFlag) { if (!completedFlag.get()) { @@ -202,12 +204,14 @@ public class IPythonClient { asyncStub.stop(request, null); } + public static void main(String[] args) { IPythonClient client = new IPythonClient("localhost", 50053); client.status(StatusRequest.newBuilder().build()); - ExecuteResponse response = - client.block_execute(ExecuteRequest.newBuilder().setCode("abcd=2").build()); + ExecuteResponse response = client.block_execute(ExecuteRequest.newBuilder(). + setCode("abcd=2").build()); System.out.println(response.getOutput()); + } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java index aebed5c..3c646ae 100644 --- a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java @@ -18,16 +18,6 @@ package org.apache.zeppelin.python; import io.grpc.ManagedChannelBuilder; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Properties; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.DefaultExecutor; import org.apache.commons.exec.ExecuteException; @@ -62,7 +52,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import py4j.GatewayServer; -/** IPython Interpreter for Zeppelin */ +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * IPython Interpreter for Zeppelin + */ public class IPythonInterpreter extends Interpreter implements ExecuteResultHandler { private static final Logger LOGGER = LoggerFactory.getLogger(IPythonInterpreter.class); @@ -87,8 +90,8 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand } /** - * Sub class can customize the interpreter by adding more python packages under PYTHONPATH. e.g. - * PySparkInterpreter + * Sub class can customize the interpreter by adding more python packages under PYTHONPATH. + * e.g. PySparkInterpreter * * @param additionalPythonPath */ @@ -98,8 +101,8 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand } /** - * Sub class can customize the interpreter by running additional python init code. e.g. - * PySparkInterpreter + * Sub class can customize the interpreter by running additional python init code. + * e.g. PySparkInterpreter * * @param additionalPythonInitFile */ @@ -128,22 +131,18 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand LOGGER.info("Python Exec: " + pythonExecutable); String checkPrerequisiteResult = checkIPythonPrerequisite(pythonExecutable); if (!StringUtils.isEmpty(checkPrerequisiteResult)) { - throw new InterpreterException( - "IPython prerequisite is not meet: " + checkPrerequisiteResult); + throw new InterpreterException("IPython prerequisite is not meet: " + + checkPrerequisiteResult); } - ipythonLaunchTimeout = - Long.parseLong(getProperty("zeppelin.ipython.launch.timeout", "30000")); + ipythonLaunchTimeout = Long.parseLong( + getProperty("zeppelin.ipython.launch.timeout", "30000")); this.zeppelinContext = buildZeppelinContext(); int ipythonPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); int jvmGatewayPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); - int message_size = - Integer.parseInt( - getProperty("zeppelin.ipython.grpc.message_size", 32 * 1024 * 1024 + "")); - ipythonClient = - new IPythonClient( - ManagedChannelBuilder.forAddress("127.0.0.1", ipythonPort) - .usePlaintext(true) - .maxInboundMessageSize(message_size)); + int message_size = Integer.parseInt(getProperty("zeppelin.ipython.grpc.message_size", + 32 * 1024 * 1024 + "")); + ipythonClient = new IPythonClient(ManagedChannelBuilder.forAddress("127.0.0.1", ipythonPort) + .usePlaintext(true).maxInboundMessageSize(message_size)); this.usePy4JAuth = Boolean.parseBoolean(getProperty("zeppelin.py4j.useAuth", "true")); this.secret = PythonUtils.createSecret(256); launchIPythonKernel(ipythonPort); @@ -154,8 +153,8 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand } /** - * non-empty return value mean the errors when checking ipython prerequisite. empty value mean - * IPython prerequisite is meet. + * non-empty return value mean the errors when checking ipython prerequisite. + * empty value mean IPython prerequisite is meet. * * @param pythonExec * @return @@ -171,7 +170,8 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand Process proc = processBuilder.start(); int ret = proc.waitFor(); if (ret != 0) { - return "Fail to run pip freeze.\n" + IOUtils.toString(new FileInputStream(stderrFile)); + return "Fail to run pip freeze.\n" + + IOUtils.toString(new FileInputStream(stderrFile)); } String freezeOutput = IOUtils.toString(new FileInputStream(stdoutFile)); if (!freezeOutput.contains("jupyter-client=")) { @@ -206,34 +206,26 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand InputStream input = getClass().getClassLoader().getResourceAsStream("grpc/python/zeppelin_python.py"); List<String> lines = IOUtils.readLines(input); - ExecuteResponse response = - ipythonClient.block_execute( - ExecuteRequest.newBuilder() - .setCode( - StringUtils.join(lines, System.lineSeparator()) - .replace("${JVM_GATEWAY_PORT}", jvmGatewayPort + "") - .replace("${JVM_GATEWAY_ADDRESS}", serverAddress)) - .build()); + ExecuteResponse response = ipythonClient.block_execute(ExecuteRequest.newBuilder() + .setCode(StringUtils.join(lines, System.lineSeparator()) + .replace("${JVM_GATEWAY_PORT}", jvmGatewayPort + "") + .replace("${JVM_GATEWAY_ADDRESS}", serverAddress)).build()); if (response.getStatus() == ExecuteStatus.ERROR) { throw new IOException("Fail to setup JVMGateway\n" + response.getOutput()); } - input = getClass().getClassLoader().getResourceAsStream("python/zeppelin_context.py"); + input = + getClass().getClassLoader().getResourceAsStream("python/zeppelin_context.py"); lines = IOUtils.readLines(input); - response = - ipythonClient.block_execute( - ExecuteRequest.newBuilder() - .setCode(StringUtils.join(lines, System.lineSeparator())) - .build()); + response = ipythonClient.block_execute(ExecuteRequest.newBuilder() + .setCode(StringUtils.join(lines, System.lineSeparator())).build()); if (response.getStatus() == ExecuteStatus.ERROR) { throw new IOException("Fail to import ZeppelinContext\n" + response.getOutput()); } - response = - ipythonClient.block_execute( - ExecuteRequest.newBuilder() - .setCode("z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext(), gateway)") - .build()); + response = ipythonClient.block_execute(ExecuteRequest.newBuilder() + .setCode("z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext(), gateway)") + .build()); if (response.getStatus() == ExecuteStatus.ERROR) { throw new IOException("Fail to setup ZeppelinContext\n" + response.getOutput()); } @@ -241,31 +233,27 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand if (additionalPythonInitFile != null) { input = getClass().getClassLoader().getResourceAsStream(additionalPythonInitFile); lines = IOUtils.readLines(input); - response = - ipythonClient.block_execute( - ExecuteRequest.newBuilder() - .setCode( - StringUtils.join(lines, System.lineSeparator()) - .replace("${JVM_GATEWAY_PORT}", jvmGatewayPort + "") - .replace("${JVM_GATEWAY_ADDRESS}", serverAddress)) - .build()); + response = ipythonClient.block_execute(ExecuteRequest.newBuilder() + .setCode(StringUtils.join(lines, System.lineSeparator()) + .replace("${JVM_GATEWAY_PORT}", jvmGatewayPort + "") + .replace("${JVM_GATEWAY_ADDRESS}", serverAddress)).build()); if (response.getStatus() == ExecuteStatus.ERROR) { - throw new IOException( - "Fail to run additional Python init file: " - + additionalPythonInitFile - + "\n" - + response.getOutput()); + throw new IOException("Fail to run additional Python init file: " + + additionalPythonInitFile + "\n" + response.getOutput()); } } } - private void launchIPythonKernel(int ipythonPort) throws IOException { + + private void launchIPythonKernel(int ipythonPort) + throws IOException { LOGGER.info("Launching IPython Kernel at port: " + ipythonPort); // copy the python scripts to a temp directory, then launch ipython kernel in that folder File pythonWorkDir = Files.createTempDirectory("zeppelin_ipython").toFile(); String[] ipythonScripts = {"ipython_server.py", "ipython_pb2.py", "ipython_pb2_grpc.py"}; for (String ipythonScript : ipythonScripts) { - URL url = getClass().getClassLoader().getResource("grpc/python" + "/" + ipythonScript); + URL url = getClass().getClassLoader().getResource("grpc/python" + + "/" + ipythonScript); FileUtils.copyURLToFile(url, new File(pythonWorkDir, ipythonScript)); } @@ -279,10 +267,10 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand executor.setWatchdog(watchDog); if (useBuiltinPy4j) { - // TODO(zjffdu) don't do hard code on py4j here + //TODO(zjffdu) don't do hard code on py4j here File py4jDestFile = new File(pythonWorkDir, "py4j-src-0.10.7.zip"); - FileUtils.copyURLToFile( - getClass().getClassLoader().getResource("python/py4j-src-0.10.7.zip"), py4jDestFile); + FileUtils.copyURLToFile(getClass().getClassLoader().getResource( + "python/py4j-src-0.10.7.zip"), py4jDestFile); if (additionalPythonPath != null) { // put the py4j at the end, because additionalPythonPath may already contain py4j. // e.g. PySparkInterpreter @@ -318,8 +306,8 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand } if ((System.currentTimeMillis() - startTime) > ipythonLaunchTimeout) { - throw new IOException( - "Fail to launch IPython Kernel in " + ipythonLaunchTimeout / 1000 + " seconds"); + throw new IOException("Fail to launch IPython Kernel in " + ipythonLaunchTimeout / 1000 + + " seconds"); } } } @@ -357,15 +345,15 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand zeppelinContext.setInterpreterContext(context); interpreterOutput.setInterpreterOutput(context.out); ExecuteResponse response = - ipythonClient.stream_execute( - ExecuteRequest.newBuilder().setCode(st).build(), interpreterOutput); + ipythonClient.stream_execute(ExecuteRequest.newBuilder().setCode(st).build(), + interpreterOutput); try { interpreterOutput.getInterpreterOutput().flush(); } catch (IOException e) { throw new RuntimeException("Fail to write output", e); } - InterpreterResult result = - new InterpreterResult(InterpreterResult.Code.valueOf(response.getStatus().name())); + InterpreterResult result = new InterpreterResult( + InterpreterResult.Code.valueOf(response.getStatus().name())); return result; } @@ -385,17 +373,14 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand } @Override - public List<InterpreterCompletion> completion( - String buf, int cursor, InterpreterContext interpreterContext) { + public List<InterpreterCompletion> completion(String buf, int cursor, + InterpreterContext interpreterContext) { LOGGER.debug("Call completion for: " + buf); List<InterpreterCompletion> completions = new ArrayList<>(); CompletionResponse response = ipythonClient.complete( - CompletionRequest.getDefaultInstance() - .newBuilder() - .setCode(buf) - .setCursor(cursor) - .build()); + CompletionRequest.getDefaultInstance().newBuilder().setCode(buf) + .setCursor(cursor).build()); for (int i = 0; i < response.getMatchesCount(); i++) { String match = response.getMatches(i); int lastIndexOfDot = match.lastIndexOf(".");
