[42/50] [abbrv] incubator-livy git commit: LIVY-342. Create Livy UI: Create Web UI Servlet and All Sessions Page (#319)

2017-06-27 Thread jshao 
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/61b206e0/server/src/main/resources/com/cloudera/livy/server/ui/static/jquery-3.2.1.min.js
--
diff --git 
a/server/src/main/resources/com/cloudera/livy/server/ui/static/jquery-3.2.1.min.js
 
b/server/src/main/resources/com/cloudera/livy/server/ui/static/jquery-3.2.1.min.js
new file mode 100644
index 000..644d35e
--- /dev/null
+++ 
b/server/src/main/resources/com/cloudera/livy/server/ui/static/jquery-3.2.1.min.js
@@ -0,0 +1,4 @@
+/*! jQuery v3.2.1 | (c) JS Foundation and other contributors | 
jquery.org/license */
+!function(a,b){"use strict";"object"==typeof module&&"object"==typeof 
module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw
 new Error("jQuery requires a window with a document");return 
b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){"use 
strict";var 
c=[],d=a.document,e=Object.getPrototypeOf,f=c.slice,g=c.concat,h=c.push,i=c.indexOf,j={},k=j.toString,l=j.hasOwnProperty,m=l.toString,n=m.call(Object),o={};function
 p(a,b){b=b||d;var 
c=b.createElement("script");c.text=a,b.head.appendChild(c).parentNode.removeChild(c)}var
 q="3.2.1",r=function(a,b){return new 
r.fn.init(a,b)},s=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,t=/^-ms-/,u=/-([a-z])/g,v=function(a,b){return
 
b.toUpperCase()};r.fn=r.prototype={jquery:q,constructor:r,length:0,toArray:function(){return
 f.call(this)},get:function(a){return 
null==a?f.call(this):a<0?this[a+this.length]:this[a]},pushStack:function(a){var 
b=r.merge(this.constructor(),a);return 
b.prevObject=this,b},each:function(a){retu
 rn r.each(this,a)},map:function(a){return 
this.pushStack(r.map(this,function(b,c){return 
a.call(b,c,b)}))},slice:function(){return 
this.pushStack(f.apply(this,arguments))},first:function(){return 
this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var 
b=this.length,c=+a+(a<0?b:0);return 
this.pushStack(c>=0&0& in a)}var x=function(a){var 
b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new 
Date,v=a.document,w=0,x=0,y=ha(),z=ha(),A=ha(),B=function(a,b){return 
a===b&&(l=!0),0},C={}.hasOwnProperty,D=[],E=D.pop,F=D.push,G=D.push,H=D.slice,I=function(a,b){for(var
 c=0,d=a.length;c

[37/50] [abbrv] incubator-livy git commit: LIVY-340. Add YARN queue support to interactive sessions. (#314)

2017-06-27 Thread jshao 
LIVY-340. Add YARN queue support to interactive sessions. (#314)



Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/7212e3f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/7212e3f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/7212e3f3

Branch: refs/heads/master
Commit: 7212e3f3bbed07ab8ea372cd7a56ee1cd2b56693
Parents: 511a05f
Author: Praveen Kanamarlapudi 
Authored: Tue Apr 18 13:02:23 2017 -0700
Committer: Alex Man 
Committed: Tue Apr 18 13:02:23 2017 -0700

--
 .../com/cloudera/livy/server/interactive/InteractiveSession.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/7212e3f3/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala
--
diff --git 
a/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala
 
b/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala
index b53dea7..6d10c57 100644
--- 
a/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala
+++ 
b/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala
@@ -88,7 +88,8 @@ object InteractiveSession extends Logging {
 SparkLauncher.EXECUTOR_CORES -> request.executorCores.map(_.toString),
 SparkLauncher.EXECUTOR_MEMORY -> 
request.executorMemory.map(_.toString),
 "spark.executor.instances" -> request.numExecutors.map(_.toString),
-"spark.app.name" -> request.name.map(_.toString)
+"spark.app.name" -> request.name.map(_.toString),
+"spark.yarn.queue" -> request.queue
   )
 
   userOpts.foreach { case (key, opt) =>

[17/50] [abbrv] incubator-livy git commit: Fix issue for some parent poms (#295)

2017-06-27 Thread jshao 
Fix issue for some parent poms (#295)

Change-Id: Ibf549db71b0f6dfc2004c3ff602e92860a38a093

Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/14c29e4d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/14c29e4d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/14c29e4d

Branch: refs/heads/master
Commit: 14c29e4d425f44d1f3caad1469a2ea4e136f0f44
Parents: ed1c86e
Author: Saisai Shao 
Authored: Mon Feb 20 16:37:26 2017 +0800
Committer: Jeff Zhang 
Committed: Mon Feb 20 16:37:26 2017 +0800

--
 core/pom.xml| 16 ---
 core/scala-2.10/pom.xml | 17 
 core/scala-2.11/pom.xml | 17 
 core/src/main/resources/build.marker|  0
 .../src/main/resources/build.marker |  0
 pom.xml | 44 +++-
 repl/src/main/resources/build.marker|  0
 .../cloudera/livy/repl/ProcessInterpreter.scala |  2 +-
 .../com/cloudera/livy/repl/ReplDriver.scala |  2 +-
 .../cloudera/livy/repl/SparkRInterpreter.scala  |  2 +-
 .../livy/repl/PythonInterpreterSpec.scala   |  2 +-
 scala-api/src/main/resources/build.marker   |  0
 scala/pom.xml   | 13 --
 13 files changed, 72 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/14c29e4d/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index ae2be28..75da834 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -68,20 +68,4 @@
 
   
 
-  
-
-  
-org.apache.maven.plugins
-maven-jar-plugin
-
-  
-
-  test-jar
-
-  
-
-  
-
-  
-
 

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/14c29e4d/core/scala-2.10/pom.xml
--
diff --git a/core/scala-2.10/pom.xml b/core/scala-2.10/pom.xml
index 27fbcba..6ac60d6 100644
--- a/core/scala-2.10/pom.xml
+++ b/core/scala-2.10/pom.xml
@@ -34,4 +34,21 @@
 ${scala-2.10.version}
 2.10
   
+
+  
+
+  
+org.apache.maven.plugins
+maven-jar-plugin
+
+  
+
+  test-jar
+
+  
+
+  
+
+  
+
 

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/14c29e4d/core/scala-2.11/pom.xml
--
diff --git a/core/scala-2.11/pom.xml b/core/scala-2.11/pom.xml
index afb0c77..e9e57df 100644
--- a/core/scala-2.11/pom.xml
+++ b/core/scala-2.11/pom.xml
@@ -34,4 +34,21 @@
 ${scala-2.11.version}
 2.11
   
+
+  
+
+  
+org.apache.maven.plugins
+maven-jar-plugin
+
+  
+
+  test-jar
+
+  
+
+  
+
+  
+
 

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/14c29e4d/core/src/main/resources/build.marker
--
diff --git a/core/src/main/resources/build.marker 
b/core/src/main/resources/build.marker
new file mode 100644
index 000..e69de29

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/14c29e4d/integration-test/minicluster-dependencies/src/main/resources/build.marker
--
diff --git 
a/integration-test/minicluster-dependencies/src/main/resources/build.marker 
b/integration-test/minicluster-dependencies/src/main/resources/build.marker
new file mode 100644
index 000..e69de29

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/14c29e4d/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 8868d35..fbc4d84 100644
--- a/pom.xml
+++ b/pom.xml
@@ -180,18 +180,23 @@
 assembly
 client-common
 client-http
+core
 core/scala-2.10
 core/scala-2.11
 coverage
 examples
 python-api
+repl
 repl/scala-2.10
 repl/scala-2.11
 rsc
+scala
+scala-api
 scala-api/scala-2.10
 scala-api/scala-2.11
 server
 test-lib
+integration-test/minicluster-dependencies
 integration-test/minicluster-dependencies/scala-2.10
 integration-test/minicluster-dependencies/scala-2.11
 integration-test
@@ -622,6 +627,13 @@
 testCompile
   
 
+
+  scala-compile-first
+  process-resources
+  
+compile
+  
+

[27/50] [abbrv] incubator-livy git commit: LIVY-272. Support Statement progress for interactive session. (#260)

2017-06-27 Thread jshao 
LIVY-272. Support Statement progress for interactive session. (#260)



Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/70f23b90
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/70f23b90
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/70f23b90

Branch: refs/heads/master
Commit: 70f23b90f9cc7fbe98663551a4cdff86a7c069dd
Parents: 126b57e
Author: Saisai Shao 
Authored: Sun Mar 12 10:16:52 2017 +0800
Committer: Alex Man 
Committed: Sat Mar 11 18:16:52 2017 -0800

--
 .../cloudera/livy/repl/SparkInterpreter.scala   |   4 +-
 .../livy/repl/SparkInterpreterSpec.scala|   2 +-
 .../cloudera/livy/repl/SparkInterpreter.scala   |   4 +-
 .../livy/repl/SparkInterpreterSpec.scala|   2 +-
 .../livy/repl/AbstractSparkInterpreter.scala|  11 +-
 .../com/cloudera/livy/repl/Interpreter.scala|  16 +-
 .../cloudera/livy/repl/ProcessInterpreter.scala |  13 +-
 .../cloudera/livy/repl/PythonInterpreter.scala  |  12 +-
 .../com/cloudera/livy/repl/ReplDriver.scala |  15 +-
 .../scala/com/cloudera/livy/repl/Session.scala  |   3 +-
 .../cloudera/livy/repl/SparkRInterpreter.scala  |  13 +-
 .../livy/repl/StatementProgressListener.scala   | 162 +
 .../livy/repl/PythonInterpreterSpec.scala   |   7 +-
 .../cloudera/livy/repl/PythonSessionSpec.scala  |   7 +-
 .../livy/repl/ScalaInterpreterSpec.scala|   5 +-
 .../livy/repl/SparkRInterpreterSpec.scala   |   5 +-
 .../cloudera/livy/repl/SparkRSessionSpec.scala  |   5 +-
 .../cloudera/livy/repl/SparkSessionSpec.scala   |   6 +-
 .../repl/StatementProgressListenerSpec.scala| 227 +++
 .../java/com/cloudera/livy/rsc/RSCConf.java |   4 +
 .../com/cloudera/livy/rsc/driver/Statement.java |  10 +
 .../InteractiveSessionServletSpec.scala |   1 +
 .../interactive/InteractiveSessionSpec.scala|  16 ++
 23 files changed, 509 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/70f23b90/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
--
diff --git 
a/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala 
b/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
index 3322de1..ec12929 100644
--- 
a/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
+++ 
b/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
@@ -33,7 +33,8 @@ import org.apache.spark.repl.SparkIMain
 /**
  * This represents a Spark interpreter. It is not thread safe.
  */
-class SparkInterpreter(conf: SparkConf)
+class SparkInterpreter(conf: SparkConf,
+override val statementProgressListener: StatementProgressListener)
   extends AbstractSparkInterpreter with SparkContextInitializer {
 
   private var sparkIMain: SparkIMain = _
@@ -103,6 +104,7 @@ class SparkInterpreter(conf: SparkConf)
   createSparkContext(conf)
 }
 
+sparkContext.addSparkListener(statementProgressListener)
 sparkContext
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/70f23b90/repl/scala-2.10/src/test/scala/com/cloudera/livy/repl/SparkInterpreterSpec.scala
--
diff --git 
a/repl/scala-2.10/src/test/scala/com/cloudera/livy/repl/SparkInterpreterSpec.scala
 
b/repl/scala-2.10/src/test/scala/com/cloudera/livy/repl/SparkInterpreterSpec.scala
index e2b783a..3df35b5 100644
--- 
a/repl/scala-2.10/src/test/scala/com/cloudera/livy/repl/SparkInterpreterSpec.scala
+++ 
b/repl/scala-2.10/src/test/scala/com/cloudera/livy/repl/SparkInterpreterSpec.scala
@@ -24,7 +24,7 @@ import com.cloudera.livy.LivyBaseUnitTestSuite
 
 class SparkInterpreterSpec extends FunSpec with Matchers with 
LivyBaseUnitTestSuite {
   describe("SparkInterpreter") {
-val interpreter = new SparkInterpreter(null)
+val interpreter = new SparkInterpreter(null, null)
 
 it("should parse Scala compile error.") {
   // Regression test for LIVY-260.

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/70f23b90/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
--
diff --git 
a/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala 
b/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
index 2bf6347..bf2f680 100644
--- 
a/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
+++ 
b/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
@@ -33,7 +33,8 @@ import

[50/50] [abbrv] incubator-livy git commit: Livy:337 Binding RPCServer to user provided port and not random port (#334)

2017-06-27 Thread jshao 
Livy:337 Binding RPCServer to user provided port and not random port (#334)

* Code changes in RPCserver for user provided port

* Indentation Changes

* Indentation Changes

* Indentation Changes

* Indentation Changes

* Configuring Port Range

* Documentation Changed

* launcher.port.range will take care of launching RPC

* Checkstyle changes

* Checkstyle changes

* Dummy push

* Code changes

* Changed BindException Handling to SocketException Handling

* Changed Import Order

* Code changes to increase port range

* Set Port isConntect to true

* Indentation Changes & port range in livy-client.conf.template

* Indentation changes

* Changed visibilty of method private

* Indentation Changes

* Indenetation Changes

* Unit test case to test port range

* Checkstyle changes

* Unit test case for port range

* Added comment for Port Range Configuration and increase port range for unit 
test case


Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/9ae24d08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/9ae24d08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/9ae24d08

Branch: refs/heads/master
Commit: 9ae24d08738652ba5fd817780711d01b110d74a9
Parents: 02eef9a
Author: pralabhkumar 
Authored: Thu Jun 8 13:22:25 2017 +0530
Committer: Jeff Zhang 
Committed: Thu Jun 8 15:52:25 2017 +0800

--
 conf/livy-client.conf.template  |  5 +-
 .../java/com/cloudera/livy/rsc/RSCConf.java |  4 +-
 .../com/cloudera/livy/rsc/rpc/RpcServer.java| 86 
 .../java/com/cloudera/livy/rsc/rpc/TestRpc.java | 27 ++
 4 files changed, 104 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/9ae24d08/conf/livy-client.conf.template
--
diff --git a/conf/livy-client.conf.template b/conf/livy-client.conf.template
index 2a92b57..06ad653 100644
--- a/conf/livy-client.conf.template
+++ b/conf/livy-client.conf.template
@@ -55,7 +55,8 @@
 
 # Address for the RSC driver to connect back with it's connection info.
 # livy.rsc.launcher.address =
-# livy.rsc.launcher.port = -1
+# Port Range on which RPC will launch . Port range in inclusive of start and 
end port .
+# livy.rsc.launcher.port.range = 1~10110
 
 # How long will the RSC wait for a connection for a Livy server before 
shutting itself down.
 # livy.rsc.server.idle-timeout = 10m
@@ -83,4 +84,4 @@
 # livy.rsc.job-cancel.timeout = 30s
 
 # Number of statements kept in driver's memory
-# livy.rsc.retained-statements = 100
\ No newline at end of file
+# livy.rsc.retained-statements = 100

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/9ae24d08/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java
--
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java 
b/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java
index d1b8b39..afd935d 100644
--- a/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java
+++ b/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java
@@ -51,8 +51,10 @@ public class RSCConf extends ClientConf {
 
 // Address for the RSC driver to connect back with it's connection info.
 LAUNCHER_ADDRESS("launcher.address", null),
+LAUNCHER_PORT_RANGE("launcher.port.range", "1~10010"),
+// Setting up of this propety by user has no benefit. It is currently 
being used
+// to pass  port information from ContextLauncher to RSCDriver
 LAUNCHER_PORT("launcher.port", -1),
-
 // How long will the RSC wait for a connection for a Livy server before 
shutting itself down.
 SERVER_IDLE_TIMEOUT("server.idle-timeout", "10m"),
 

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/9ae24d08/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcServer.java
--
diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcServer.java 
b/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcServer.java
index 1d3e6c5..44db976 100644
--- a/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcServer.java
+++ b/rsc/src/main/java/com/cloudera/livy/rsc/rpc/RpcServer.java
@@ -19,7 +19,10 @@ package com.cloudera.livy.rsc.rpc;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.net.BindException;
 import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.SocketException;
 import java.security.SecureRandom;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -61,18 +64,78 @@ public class RpcServer implements Closeable {
   private static final SecureRandom RND = new SecureRandom();
 
   private final String

[08/50] [abbrv] incubator-livy git commit: Bump to 0.4.0-SNAPSHOT for master branch (#282)

2017-06-27 Thread jshao 
Bump to 0.4.0-SNAPSHOT for master branch (#282)

Change-Id: I32dd970bfa1150e3b47ecbfd7c1a7c348bb7e259

Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/9575d8f7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/9575d8f7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/9575d8f7

Branch: refs/heads/master
Commit: 9575d8f7597c55918635760da6d6916720f5dad6
Parents: 34f6c7f
Author: Saisai Shao 
Authored: Fri Feb 10 12:07:19 2017 +0800
Committer: Jeff Zhang 
Committed: Fri Feb 10 12:07:19 2017 +0800

--
 api/pom.xml  | 4 ++--
 assembly/pom.xml | 4 ++--
 client-common/pom.xml| 4 ++--
 client-http/pom.xml  | 4 ++--
 core/pom.xml | 4 ++--
 core/scala-2.10/pom.xml  | 4 ++--
 core/scala-2.11/pom.xml  | 4 ++--
 coverage/pom.xml | 4 ++--
 examples/pom.xml | 4 ++--
 integration-test/minicluster-dependencies/pom.xml| 4 ++--
 integration-test/minicluster-dependencies/scala-2.10/pom.xml | 4 ++--
 integration-test/minicluster-dependencies/scala-2.11/pom.xml | 4 ++--
 integration-test/pom.xml | 4 ++--
 pom.xml  | 2 +-
 python-api/pom.xml   | 4 ++--
 python-api/setup.py  | 2 +-
 repl/pom.xml | 4 ++--
 repl/scala-2.10/pom.xml  | 4 ++--
 repl/scala-2.11/pom.xml  | 4 ++--
 rsc/pom.xml  | 4 ++--
 scala-api/pom.xml| 4 ++--
 scala-api/scala-2.10/pom.xml | 4 ++--
 scala-api/scala-2.11/pom.xml | 4 ++--
 scala/pom.xml| 4 ++--
 server/pom.xml   | 4 ++--
 test-lib/pom.xml | 4 ++--
 26 files changed, 50 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/9575d8f7/api/pom.xml
--
diff --git a/api/pom.xml b/api/pom.xml
index d196cc5..156d987 100644
--- a/api/pom.xml
+++ b/api/pom.xml
@@ -20,12 +20,12 @@
   
 com.cloudera.livy
 livy-main
-0.3.0-SNAPSHOT
+0.4.0-SNAPSHOT
   
 
   com.cloudera.livy
   livy-api
-  0.3.0-SNAPSHOT
+  0.4.0-SNAPSHOT
   jar
 
   

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/9575d8f7/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index e9df377..87da4e5 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,12 +21,12 @@
   
 com.cloudera.livy
 livy-main
-0.3.0-SNAPSHOT
+0.4.0-SNAPSHOT
 ../pom.xml
   
 
   livy-assembly
-  0.3.0-SNAPSHOT
+  0.4.0-SNAPSHOT
   pom
 
   

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/9575d8f7/client-common/pom.xml
--
diff --git a/client-common/pom.xml b/client-common/pom.xml
index 810be59..9cc713c 100644
--- a/client-common/pom.xml
+++ b/client-common/pom.xml
@@ -21,12 +21,12 @@
   
 com.cloudera.livy
 livy-main
-0.3.0-SNAPSHOT
+0.4.0-SNAPSHOT
   
 
   com.cloudera.livy
   livy-client-common
-  0.3.0-SNAPSHOT
+  0.4.0-SNAPSHOT
   jar
 
   

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/9575d8f7/client-http/pom.xml
--
diff --git a/client-http/pom.xml b/client-http/pom.xml
index 4c22ddb..478e222 100644
--- a/client-http/pom.xml
+++ b/client-http/pom.xml
@@ -21,12 +21,12 @@
   
 com.cloudera.livy
 livy-main
-0.3.0-SNAPSHOT
+0.4.0-SNAPSHOT
   
 
   com.cloudera.livy
   livy-client-http
-  0.3.0-SNAPSHOT
+  0.4.0-SNAPSHOT
   jar
 
   

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/9575d8f7/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index bfac5fd..ae2be28 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -23,12 +23,12 @@
   
 com.cloudera.livy
 multi-scala-project-root
-0.3.0-SNAPSHOT
+0.4.0-SNAPSHOT

[04/50] [abbrv] incubator-livy git commit: Merge pull request #237 from jerryshao/LIVY-255

2017-06-27 Thread jshao 
Merge pull request #237 from jerryshao/LIVY-255

LIVY-255. Update the docs for Livy build and Spark2 support

Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/dbdad9d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/dbdad9d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/dbdad9d3

Branch: refs/heads/revert-237-LIVY-255
Commit: dbdad9d37dfdf131b735df44b79a82837e53db5a
Parents: b8a0839 2e812a7
Author: Jeff Zhang 
Authored: Wed Nov 30 11:01:43 2016 +0800
Committer: GitHub 
Committed: Wed Nov 30 11:01:43 2016 +0800

--
 README.rst | 25 +
 1 file changed, 13 insertions(+), 12 deletions(-)
--

[46/50] [abbrv] incubator-livy git commit: Minor. Add session information log when session is created (#325)

2017-06-27 Thread jshao 
Minor. Add session information log when session is created (#325)

* Add session information log when session is created

Change-Id: I8702ebe1d893cf328b4490e5b5d09f3afd02b7ce

* add more logs

Change-Id: I969edf4e680e59e3ecfbaa50431ae61bd03d4795

* simplify the code

Change-Id: If2c7876ddb6020ace3872c9ae639d69d58b02f48

* Address the comments

Change-Id: Iba5f10ad73ff67b97af8292769b254346b8ef7c8


Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/d51e9982
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/d51e9982
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/d51e9982

Branch: refs/heads/master
Commit: d51e998204e3a10f9207280951f98edc8755c433
Parents: 61b206e
Author: Saisai Shao 
Authored: Fri May 12 08:36:19 2017 +0800
Committer: Jeff Zhang 
Committed: Thu May 11 17:36:19 2017 -0700

--
 .../cloudera/livy/server/batch/BatchSession.scala | 11 ---
 .../livy/server/batch/CreateBatchRequest.scala| 17 +
 .../interactive/CreateInteractiveRequest.scala| 18 ++
 .../server/interactive/InteractiveSession.scala   |  5 -
 4 files changed, 47 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d51e9982/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala
--
diff --git 
a/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala 
b/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala
index 452a9d8..1a097b7 100644
--- a/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala
+++ b/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala
@@ -25,7 +25,7 @@ import scala.util.Random
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 
-import com.cloudera.livy.LivyConf
+import com.cloudera.livy.{LivyConf, Logging}
 import com.cloudera.livy.server.recovery.SessionStore
 import com.cloudera.livy.sessions.{Session, SessionState}
 import com.cloudera.livy.sessions.Session._
@@ -41,7 +41,7 @@ case class BatchRecoveryMetadata(
 version: Int = 1)
   extends RecoveryMetadata
 
-object BatchSession {
+object BatchSession extends Logging {
   val RECOVERY_SESSION_TYPE = "batch"
 
   def create(
@@ -89,6 +89,8 @@ object BatchSession {
   SparkApp.create(appTag, None, Option(sparkSubmit), livyConf, Option(s))
 }
 
+info(s"Creating batch session $id: [owner: $owner, request: $request]")
+
 new BatchSession(
   id,
   appTag,
@@ -153,7 +155,10 @@ class BatchSession(
 synchronized {
   debug(s"$this state changed from $oldState to $newState")
   newState match {
-case SparkApp.State.RUNNING => _state = SessionState.Running()
+case SparkApp.State.RUNNING =>
+  _state = SessionState.Running()
+  info(s"Batch session $id created [appid: ${appId.orNull}, state: 
${state.toString}, " +
+s"info: ${appInfo.asJavaMap}]")
 case SparkApp.State.FINISHED => _state = SessionState.Success()
 case SparkApp.State.KILLED | SparkApp.State.FAILED =>
   _state = SessionState.Dead()

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d51e9982/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala
--
diff --git 
a/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala 
b/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala
index 99459d5..29c269c 100644
--- 
a/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala
+++ 
b/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala
@@ -37,4 +37,21 @@ class CreateBatchRequest {
   var name: Option[String] = None
   var conf: Map[String, String] = Map()
 
+  override def toString: String = {
+s"[proxyUser: $proxyUser, " +
+  s"file: $file, " +
+  (if (args.nonEmpty) s"args: ${args.mkString(",")}, " else "") +
+  (if (jars.nonEmpty) s"jars: ${jars.mkString(",")}, " else "") +
+  (if (pyFiles.nonEmpty) s"pyFiles: ${pyFiles.mkString(",")}, " else "") +
+  (if (files.nonEmpty) s"files: ${files.mkString(",")}, " else "") +
+  (if (archives.nonEmpty) s"archives: ${archives.mkString(",")}, " else 
"") +
+  (if (driverMemory.isDefined) s"driverMemory: ${driverMemory.get}, " else 
"") +
+  (if (driverCores.isDefined) s"driverCores: ${driverCores.get}, " else 
"") +
+  (if (executorMemory.isDefined) s"executorMemory: ${executorMemory.get}, 
" else "") +
+  (if (executorCores.isDefined) s"executorCores: ${executorCores.get}, " 
else "") +
+

[31/50] [abbrv] incubator-livy git commit: Make several modules push-able to repo (#305)

2017-06-27 Thread jshao 
Make several modules push-able to repo (#305)

Change-Id: I9daec6ac7fdb2c9f727cbab6db042ec144c4edb8

Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/b7cc00a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/b7cc00a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/b7cc00a9

Branch: refs/heads/master
Commit: b7cc00a947832568fed7110cb94affcdd067d375
Parents: eb7e6cc
Author: Saisai Shao 
Authored: Thu Mar 16 09:58:03 2017 +0800
Committer: Jeff Zhang 
Committed: Thu Mar 16 09:58:03 2017 +0800

--
 core/pom.xml   | 4 
 repl/pom.xml   | 4 
 server/pom.xml | 4 
 3 files changed, 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/b7cc00a9/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index 75da834..437f00f 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -31,10 +31,6 @@
   0.4.0-SNAPSHOT
   pom
 
-  
-true
-  
-
   
 
   com.cloudera.livy

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/b7cc00a9/repl/pom.xml
--
diff --git a/repl/pom.xml b/repl/pom.xml
index 12675f7..6d5f73c 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -31,10 +31,6 @@
   0.4.0-SNAPSHOT
   pom
 
-  
-true
-  
-
   
 
 

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/b7cc00a9/server/pom.xml
--
diff --git a/server/pom.xml b/server/pom.xml
index 7ca8c65..6846be9 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -31,10 +31,6 @@
   0.4.0-SNAPSHOT
   jar
 
-  
-true
-  
-

[01/50] [abbrv] incubator-livy git commit: LIVY-183. Deploy livy-main pom to maven repo.

2017-06-27 Thread jshao 
Repository: incubator-livy
Updated Branches:
  refs/heads/branch-0.2 [created] 491de2126
  refs/heads/branch-0.3 [created] 0afee2301
  refs/heads/gh-pages [created] fd20a236a
  refs/heads/master [created] 9ae24d087
  refs/heads/rel-0.2.0-refresh [created] e9256c27f
  refs/heads/revert-237-LIVY-255 [created] 2da435ad1
Updated Tags:  refs/tags/v0.2.0 [created] dc3cbcad4
  refs/tags/v0.3.0 [created] e36b6f57c


LIVY-183. Deploy livy-main pom to maven repo.

This requires changing the default value of "skipDeploy" since it
cannot be overridden just for the root pom. Ah, maven.

Closes #160

(cherry picked from commit a5a09746c162c0e3ae5fb2bf6f53023f92722f35)


Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/e9256c27
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/e9256c27
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/e9256c27

Branch: refs/heads/rel-0.2.0-refresh
Commit: e9256c27f08af5616a38f7d8a8044c6dac83ce98
Parents: dc3cbca
Author: Marcelo Vanzin 
Authored: Tue Jul 5 10:26:23 2016 -0700
Committer: Marcelo Vanzin 
Committed: Tue Jul 5 13:00:18 2016 -0700

--
 api/pom.xml  | 1 -
 assembly/pom.xml | 1 +
 client-common/pom.xml| 4 
 client-http/pom.xml  | 4 
 core/pom.xml | 4 
 integration-test/pom.xml | 1 +
 pom.xml  | 6 +++---
 repl/pom.xml | 4 
 rsc/pom.xml  | 1 -
 server/pom.xml   | 4 
 test-lib/pom.xml | 4 
 11 files changed, 21 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/e9256c27/api/pom.xml
--
diff --git a/api/pom.xml b/api/pom.xml
index e61bb96..af45309 100644
--- a/api/pom.xml
+++ b/api/pom.xml
@@ -30,7 +30,6 @@
 
   
 ${asf.copyright.header}
-false
   
 
   

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/e9256c27/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 44786d1..a483843 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -32,6 +32,7 @@
   
 livy-server-${project.version}
 zip
+true
   
 
   

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/e9256c27/client-common/pom.xml
--
diff --git a/client-common/pom.xml b/client-common/pom.xml
index 13df3b1..d985e7e 100644
--- a/client-common/pom.xml
+++ b/client-common/pom.xml
@@ -29,10 +29,6 @@
   0.2.0
   jar
 
-  
-false
-  
-
   
 
   com.cloudera.livy

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/e9256c27/client-http/pom.xml
--
diff --git a/client-http/pom.xml b/client-http/pom.xml
index f60830e..208f91b 100644
--- a/client-http/pom.xml
+++ b/client-http/pom.xml
@@ -29,10 +29,6 @@
   0.2.0
   jar
 
-  
-false
-  
-
   
 
   com.cloudera.livy

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/e9256c27/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index ba57ac4..5a9294c 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -31,6 +31,10 @@
   0.2.0
   jar
 
+  
+true
+  
+
   
 
   com.cloudera.livy

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/e9256c27/integration-test/pom.xml
--
diff --git a/integration-test/pom.xml b/integration-test/pom.xml
index f773681..a3b32ad 100644
--- a/integration-test/pom.xml
+++ b/integration-test/pom.xml
@@ -39,6 +39,7 @@
 -->
 default
 ${skipTests}
+true
   
 
   

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/e9256c27/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 9bd3fbd..e590e52 100644
--- a/pom.xml
+++ b/pom.xml
@@ -78,10 +78,10 @@
 -->
 
 
-true
+false

[35/50] [abbrv] incubator-livy git commit: LIVY-323. Fixed livy-repl doesn't start with a lot of spark packages. (#312)

2017-06-27 Thread jshao 
LIVY-323. Fixed livy-repl doesn't start with a lot of spark packages. (#312)

Some Spark packages are depending on scala-reflect 2.11.0 and it conflicts with 
Spark's scala version 2.11.8.
This's not Livy's fault actually but doesn't hurt to make Livy more fault 
tolerant.
Since the scala-reflect jar with the correct version must already be in 
CLASSPATH, fixed livy-repl to not load user supplied scala-reflect jars.

Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/07f6072f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/07f6072f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/07f6072f

Branch: refs/heads/master
Commit: 07f6072f9d90ec970a1f9fbaccf0452f3a783538
Parents: 221aa9c
Author: Alex Man 
Authored: Sat Apr 8 17:12:37 2017 -0700
Committer: GitHub 
Committed: Sat Apr 8 17:12:37 2017 -0700

--
 .../src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala | 4 
 .../src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala | 4 
 2 files changed, 8 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/07f6072f/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
--
diff --git 
a/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala 
b/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
index ec12929..5ef5491 100644
--- 
a/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
+++ 
b/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
@@ -92,6 +92,10 @@ class SparkInterpreter(conf: SparkConf,
 .filter { u => u.getProtocol == "file" && new 
File(u.getPath).isFile }
 // Livy rsc and repl are also in the extra jars list. Filter them 
out.
 .filterNot { u => 
Paths.get(u.toURI).getFileName.toString.startsWith("livy-") }
+// Some bad spark packages depend on the wrong version of 
scala-reflect. Blacklist it.
+.filterNot { u =>
+  
Paths.get(u.toURI).getFileName.toString.contains("org.scala-lang_scala-reflect")
+}
 
   extraJarPath.foreach { p => debug(s"Adding $p to Scala interpreter's 
class path...") }
   sparkIMain.addUrlsToClassPath(extraJarPath: _*)

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/07f6072f/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
--
diff --git 
a/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala 
b/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
index bf2f680..6735b3a 100644
--- 
a/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
+++ 
b/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala
@@ -78,6 +78,10 @@ class SparkInterpreter(conf: SparkConf,
 .filter { u => u.getProtocol == "file" && new 
File(u.getPath).isFile }
 // Livy rsc and repl are also in the extra jars list. Filter them 
out.
 .filterNot { u => 
Paths.get(u.toURI).getFileName.toString.startsWith("livy-") }
+// Some bad spark packages depend on the wrong version of 
scala-reflect. Blacklist it.
+.filterNot { u =>
+  
Paths.get(u.toURI).getFileName.toString.contains("org.scala-lang_scala-reflect")
+}
 
   extraJarPath.foreach { p => debug(s"Adding $p to Scala interpreter's 
class path...") }
   sparkILoop.addUrlsToClassPath(extraJarPath: _*)

[33/50] [abbrv] incubator-livy git commit: LIVY-313. Fixed SparkRInterpreter always returning success. (#307)

2017-06-27 Thread jshao 
LIVY-313. Fixed SparkRInterpreter always returning success. (#307)

* LIVY-313. Fixed SparkRInterpreter always returning success.

- Stopped redirecting stderr to stdout.
- Continue to read ErrorStream (it was only being read once).
- Checking for any errors returned by stderr before returning success.

* Fixing scalastyle check error

* Changing the way errors are handled in SparkRInterpreter

* Fixing scalastyle check error

* Updating SparkRSessionSpec


Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/0de0e286
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/0de0e286
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/0de0e286

Branch: refs/heads/master
Commit: 0de0e28658d5490e289a290b3cdf8a9f12e19eb0
Parents: 5e6f9ed
Author: Jonathan Alter 
Authored: Wed Mar 22 23:56:57 2017 -0700
Committer: Jeff Zhang 
Committed: Thu Mar 23 14:56:57 2017 +0800

--
 .../cloudera/livy/repl/SparkRInterpreter.scala  | 61 ++--
 .../livy/repl/SparkRInterpreterSpec.scala   | 13 ++---
 .../cloudera/livy/repl/SparkRSessionSpec.scala  |  8 +--
 3 files changed, 53 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0de0e286/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala
--
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala 
b/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala
index 8e5f3c0..7318b1e 100644
--- a/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala
+++ b/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala
@@ -37,9 +37,12 @@ import org.json4s.JsonDSL._
 import com.cloudera.livy.client.common.ClientConf
 import com.cloudera.livy.rsc.RSCConf
 
+private case class RequestResponse(content: String, error: Boolean)
+
 // scalastyle:off println
 object SparkRInterpreter {
   private val LIVY_END_MARKER = "LIVY_END_OF_COMMAND"
+  private val LIVY_ERROR_MARKER = "LIVY_END_OF_ERROR"
   private val PRINT_MARKER = f"""print("$LIVY_END_MARKER")"""
   private val EXPECTED_OUTPUT = f"""[1] "$LIVY_END_MARKER
 
@@ -188,18 +191,25 @@ class SparkRInterpreter(process: Process,
 }
 
 try {
-  var content: JObject = TEXT_PLAIN -> sendRequest(code)
-
-  // If we rendered anything, pass along the last image.
-  tempFile.foreach { case file =>
-val bytes = Files.readAllBytes(file)
-if (bytes.nonEmpty) {
-  val image = Base64.encodeBase64String(bytes)
-  content = content ~ (IMAGE_PNG -> image)
+  val response = sendRequest(code)
+
+  if (response.error) {
+Interpreter.ExecuteError("Error", response.content)
+  } else {
+var content: JObject = TEXT_PLAIN -> response.content
+
+// If we rendered anything, pass along the last image.
+tempFile.foreach { case file =>
+  val bytes = Files.readAllBytes(file)
+  if (bytes.nonEmpty) {
+val image = Base64.encodeBase64String(bytes)
+content = content ~ (IMAGE_PNG -> image)
+  }
 }
+
+Interpreter.ExecuteSuccess(content)
   }
 
-  Interpreter.ExecuteSuccess(content)
 } catch {
   case e: Error =>
 Interpreter.ExecuteError("Error", e.output)
@@ -211,14 +221,16 @@ class SparkRInterpreter(process: Process,
 
   }
 
-  private def sendRequest(code: String): String = {
-
stdin.println(s"""try(eval(parse(text="${StringEscapeUtils.escapeJava(code)}")))""")
+  private def sendRequest(code: String): RequestResponse = {
+
stdin.println(s"""tryCatch(eval(parse(text="${StringEscapeUtils.escapeJava(code)}"))
+ |,error = function(e) sprintf("%s%s", e, 
"${LIVY_ERROR_MARKER}"))
+  """.stripMargin)
 stdin.flush()
 
 stdin.println(PRINT_MARKER)
 stdin.flush()
 
-readTo(EXPECTED_OUTPUT)
+readTo(EXPECTED_OUTPUT, LIVY_ERROR_MARKER)
   }
 
   override protected def sendShutdownRequest() = {
@@ -242,7 +254,10 @@ class SparkRInterpreter(process: Process,
   }
 
   @tailrec
-  private def readTo(marker: String, output: StringBuilder = 
StringBuilder.newBuilder): String = {
+  private def readTo(
+  marker: String,
+  errorMarker: String,
+  output: StringBuilder = StringBuilder.newBuilder): RequestResponse = {
 var char = readChar(output)
 
 // Remove any ANSI color codes which match the pattern 
"\u001b\\[[0-9;]*[mG]".
@@ -259,15 +274,25 @@ class SparkRInterpreter(process: Process,
 }
 
 if (output.endsWith(marker)) {
-  val result = output.toString()
-  result.substring(0, result.length -

[11/50] [abbrv] incubator-livy git commit: LIVY-308. Avoid enabling Hive by default in Spark2. (#287)

2017-06-27 Thread jshao 
LIVY-308. Avoid enabling Hive by default in Spark2. (#287)

Fixed broken integration test on Spark 2.1.

Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/663c6130
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/663c6130
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/663c6130

Branch: refs/heads/master
Commit: 663c6130a5f576965b74e4448e68bc7b7668ad48
Parents: 807036a
Author: Saisai Shao 
Authored: Mon Feb 13 15:08:21 2017 +0800
Committer: Alex Man 
Committed: Mon Feb 13 02:08:21 2017 -0500

--
 .travis.yml |  2 ++
 pom.xml | 12 
 .../cloudera/livy/repl/SparkContextInitializer.scala|  2 +-
 3 files changed, 15 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/663c6130/.travis.yml
--
diff --git a/.travis.yml b/.travis.yml
index b52b0bf..021c061 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,8 +5,10 @@ language: scala
 env:
   - MVN_FLAG='-Pspark-1.6 -DskipTests'
   - MVN_FLAG='-Pspark-2.0 -DskipTests'
+  - MVN_FLAG='-Pspark-2.1 -DskipTests'
   - MVN_FLAG='-Pspark-1.6 -DskipITs'
   - MVN_FLAG='-Pspark-2.0 -DskipITs'
+  - MVN_FLAG='-Pspark-2.1 -DskipITs'
 
 jdk:
   - oraclejdk7

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/663c6130/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 28db23c..8868d35 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1005,6 +1005,18 @@
 2.0.1
   
 
+
+
+  spark-2.1
+  
+
+  spark-2.1
+
+  
+  
+2.1.0
+  
+
   
 
 

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/663c6130/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala
--
diff --git 
a/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala 
b/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala
index 33c7f37..fd082eb 100644
--- a/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala
+++ b/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala
@@ -82,7 +82,7 @@ trait SparkContextInitializer extends Logging {
 builder.getClass.getMethod("config", classOf[SparkConf]).invoke(builder, 
conf)
 
 var spark: Object = null
-if (conf.get("spark.sql.catalogImplementation", "hive").toLowerCase == 
"hive") {
+if (conf.get("spark.sql.catalogImplementation", "in-memory").toLowerCase 
== "hive") {
   if 
(sparkClz.getMethod("hiveClassesArePresent").invoke(sparkObj).asInstanceOf[Boolean])
 {
 val loader = Option(Thread.currentThread().getContextClassLoader)
   .getOrElse(getClass.getClassLoader)

[12/50] [abbrv] incubator-livy git commit: LIVY-274. Add travis CI status in README (#288)

2017-06-27 Thread jshao 
LIVY-274. Add travis CI status in README (#288)



Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/a36e44f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/a36e44f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/a36e44f4

Branch: refs/heads/master
Commit: a36e44f405dae0409815f6872762b1a38f9bffa9
Parents: 663c613
Author: Jonathan Alter 
Authored: Tue Feb 14 13:58:52 2017 -0800
Committer: Alex Man 
Committed: Tue Feb 14 16:58:52 2017 -0500

--
 README.rst | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/a36e44f4/README.rst
--
diff --git a/README.rst b/README.rst
index e0b76c2..1331bf7 100644
--- a/README.rst
+++ b/README.rst
@@ -1,6 +1,9 @@
 Welcome to Livy
 ===
 
+.. image:: https://travis-ci.org/cloudera/livy.svg?branch=master
+:target: https://travis-ci.org/cloudera/livy
+
 Livy is an open source REST interface for interacting with `Apache Spark`_ 
from anywhere.
 It supports executing snippets of code or programs in a Spark context that 
runs locally or in `Apache Hadoop YARN`_.

[02/50] [abbrv] incubator-livy git commit: Update the docs related Livy build and Spark 2 support

2017-06-27 Thread jshao 
Update the docs related Livy build and Spark 2 support

Change-Id: Id59c628e73da46a124f718b192fc92336ac8eace


Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/97251bdc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/97251bdc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/97251bdc

Branch: refs/heads/revert-237-LIVY-255
Commit: 97251bdcfa9543997d85a06149820a7c794fa8d2
Parents: b8a0839
Author: jerryshao 
Authored: Mon Nov 28 15:41:58 2016 +0800
Committer: jerryshao 
Committed: Mon Nov 28 15:41:58 2016 +0800

--
 README.rst | 25 +
 1 file changed, 13 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/97251bdc/README.rst
--
diff --git a/README.rst b/README.rst
index b23136a..1d2d2e5 100644
--- a/README.rst
+++ b/README.rst
@@ -51,8 +51,14 @@ Required python packages for building Livy:
 
 
 To run Livy, you will also need a Spark installation. You can get Spark 
releases at
-https://spark.apache.org/downloads.html. Livy requires at least Spark 1.6 and 
currently
-only supports Scala 2.10 builds of Spark.
+https://spark.apache.org/downloads.html.
+
+Livy requires at least Spark 1.6 and supports both Scala 2.10 and 2.11 builds 
of Spark, Livy
+will automatically pick repl dependencies through detecting the Scala version 
of Spark.
+
+Livy also supports Spark 2.0+ for both interactive and batch submission, you 
could seamlessly
+switch to different versions of Spark through ``SPARK_HOME`` configuration, 
without needing to
+rebuild Livy.
 
 
 Building Livy
@@ -66,16 +72,11 @@ Livy is built using `Apache Maven`_. To check out and build 
Livy, run:
 cd livy
 mvn package
 
-By default Livy is built against the CDH 5.5 distribution of Spark (based off 
Spark 1.5.0). You can
-build Livy against a different version of Spark by setting the 
``spark.version`` property:
-
-.. code:: shell
-
-mvn -Dspark.version=1.6.1 package
-
-The version of Spark used when running Livy does not need to match the version 
used to build Livy.
-The Livy package itself does not contain a Spark distribution, and will work 
with any supported
-version of Spark.
+By default Livy is built against Apache Spark 1.6.2, but the version of Spark 
used when running
+Livy does not need to match the version used to build Livy. Livy internally 
uses reflection to
+mitigate the gaps of different Spark versions, also Livy package itself does 
not
+contain a Spark distribution, so it will work with any supported version of 
Spark (Spark 1.6+)
+without to rebuild against specific version of Spark.
 
 .. _Apache Maven: http://maven.apache.org

[09/50] [abbrv] incubator-livy git commit: Bump the version of branch-0.3 to 0.3.1-SNAPSHOT. (#281)

2017-06-27 Thread jshao 
Bump the version of branch-0.3 to 0.3.1-SNAPSHOT. (#281)



Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/0afee230
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/0afee230
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/0afee230

Branch: refs/heads/branch-0.3
Commit: 0afee2301dd321888fb4cb67529f381ee494d669
Parents: e36b6f5
Author: Saisai Shao 
Authored: Sat Feb 11 00:47:55 2017 +0800
Committer: Alex Man 
Committed: Fri Feb 10 11:47:55 2017 -0500

--
 api/pom.xml  | 4 ++--
 assembly/pom.xml | 4 ++--
 client-common/pom.xml| 4 ++--
 client-http/pom.xml  | 4 ++--
 core/pom.xml | 4 ++--
 core/scala-2.10/pom.xml  | 4 ++--
 core/scala-2.11/pom.xml  | 4 ++--
 coverage/pom.xml | 4 ++--
 examples/pom.xml | 4 ++--
 integration-test/minicluster-dependencies/pom.xml| 4 ++--
 integration-test/minicluster-dependencies/scala-2.10/pom.xml | 4 ++--
 integration-test/minicluster-dependencies/scala-2.11/pom.xml | 4 ++--
 integration-test/pom.xml | 4 ++--
 pom.xml  | 2 +-
 python-api/pom.xml   | 4 ++--
 python-api/setup.py  | 2 +-
 repl/pom.xml | 4 ++--
 repl/scala-2.10/pom.xml  | 4 ++--
 repl/scala-2.11/pom.xml  | 4 ++--
 rsc/pom.xml  | 4 ++--
 scala-api/pom.xml| 4 ++--
 scala-api/scala-2.10/pom.xml | 4 ++--
 scala-api/scala-2.11/pom.xml | 4 ++--
 scala/pom.xml| 4 ++--
 server/pom.xml   | 4 ++--
 test-lib/pom.xml | 4 ++--
 26 files changed, 50 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0afee230/api/pom.xml
--
diff --git a/api/pom.xml b/api/pom.xml
index 8873c61..93f16d6 100644
--- a/api/pom.xml
+++ b/api/pom.xml
@@ -20,12 +20,12 @@
   
 com.cloudera.livy
 livy-main
-0.3.0
+0.3.1-SNAPSHOT
   
 
   com.cloudera.livy
   livy-api
-  0.3.0
+  0.3.1-SNAPSHOT
   jar
 
   

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0afee230/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index fe1515a..b54a4f7 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,12 +21,12 @@
   
 com.cloudera.livy
 livy-main
-0.3.0
+0.3.1-SNAPSHOT
 ../pom.xml
   
 
   livy-assembly
-  0.3.0
+  0.3.1-SNAPSHOT
   pom
 
   

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0afee230/client-common/pom.xml
--
diff --git a/client-common/pom.xml b/client-common/pom.xml
index c3202fb..026ed32 100644
--- a/client-common/pom.xml
+++ b/client-common/pom.xml
@@ -21,12 +21,12 @@
   
 com.cloudera.livy
 livy-main
-0.3.0
+0.3.1-SNAPSHOT
   
 
   com.cloudera.livy
   livy-client-common
-  0.3.0
+  0.3.1-SNAPSHOT
   jar
 
   

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0afee230/client-http/pom.xml
--
diff --git a/client-http/pom.xml b/client-http/pom.xml
index dd295c1..8a26983 100644
--- a/client-http/pom.xml
+++ b/client-http/pom.xml
@@ -21,12 +21,12 @@
   
 com.cloudera.livy
 livy-main
-0.3.0
+0.3.1-SNAPSHOT
   
 
   com.cloudera.livy
   livy-client-http
-  0.3.0
+  0.3.1-SNAPSHOT
   jar
 
   

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/0afee230/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index e17d871..1aa377f 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -23,12 +23,12 @@
   
 com.cloudera.livy
 multi-scala-project-root
-0.3.0
+0.3.1-SNAPSHOT
 ../scala/pom.xml
   
 
   livy-core-parent
-  0.3.0
+  0.3.1-SNAPSHOT
   pom