LIVY-375. Change Livy code package name to org.apache.livy This PR propose to change Livy code package name to org.apache.livy. This is one part to move Livy to Apache.
Author: jerryshao <ss...@hortonworks.com> Closes #12 from jerryshao/LIVY-375. Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/412ccc8f Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/412ccc8f Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/412ccc8f Branch: refs/heads/master Commit: 412ccc8fcf96854fedbe76af8e5a6fec2c542d25 Parents: cb5b8aa Author: jerryshao <ss...@hortonworks.com> Authored: Wed Jul 5 11:06:17 2017 +0800 Committer: jerryshao <ss...@hortonworks.com> Committed: Wed Jul 5 11:06:17 2017 +0800 ---------------------------------------------------------------------- README.rst | 2 +- api/pom.xml | 8 +- api/src/main/java/com/cloudera/livy/Job.java | 29 - .../main/java/com/cloudera/livy/JobContext.java | 70 --- .../main/java/com/cloudera/livy/JobHandle.java | 76 --- .../main/java/com/cloudera/livy/LivyClient.java | 109 ---- .../com/cloudera/livy/LivyClientBuilder.java | 163 ----- .../com/cloudera/livy/LivyClientFactory.java | 44 -- .../com/cloudera/livy/annotations/Private.java | 30 - api/src/main/java/org/apache/livy/Job.java | 29 + .../main/java/org/apache/livy/JobContext.java | 70 +++ .../main/java/org/apache/livy/JobHandle.java | 76 +++ .../main/java/org/apache/livy/LivyClient.java | 109 ++++ .../java/org/apache/livy/LivyClientBuilder.java | 163 +++++ .../java/org/apache/livy/LivyClientFactory.java | 44 ++ .../org/apache/livy/annotations/Private.java | 30 + .../com/cloudera/livy/TestClientFactory.java | 86 --- .../cloudera/livy/TestLivyClientBuilder.java | 81 --- .../java/org/apache/livy/TestClientFactory.java | 86 +++ .../org/apache/livy/TestLivyClientBuilder.java | 81 +++ .../com.cloudera.livy.LivyClientFactory | 1 - .../services/org.apache.livy.LivyClientFactory | 1 + assembly/pom.xml | 6 +- bin/livy-server | 2 +- checkstyle.xml | 2 +- client-common/pom.xml | 10 +- .../livy/client/common/AbstractJobHandle.java | 105 ---- .../livy/client/common/BufferUtils.java | 42 -- .../cloudera/livy/client/common/ClientConf.java | 250 -------- .../livy/client/common/HttpMessages.java | 142 ----- .../cloudera/livy/client/common/Serializer.java | 85 --- .../cloudera/livy/client/common/TestUtils.java | 79 --- .../livy/client/common/AbstractJobHandle.java | 105 ++++ .../apache/livy/client/common/BufferUtils.java | 42 ++ .../apache/livy/client/common/ClientConf.java | 250 ++++++++ .../apache/livy/client/common/HttpMessages.java | 142 +++++ .../apache/livy/client/common/Serializer.java | 85 +++ .../apache/livy/client/common/TestUtils.java | 79 +++ .../client/common/TestAbstractJobHandle.java | 92 --- .../livy/client/common/TestBufferUtils.java | 61 -- .../livy/client/common/TestClientConf.java | 230 ------- .../livy/client/common/TestHttpMessages.java | 130 ---- .../livy/client/common/TestSerializer.java | 78 --- .../livy/client/common/TestTestUtils.java | 38 -- .../client/common/TestAbstractJobHandle.java | 92 +++ .../livy/client/common/TestBufferUtils.java | 61 ++ .../livy/client/common/TestClientConf.java | 230 +++++++ .../livy/client/common/TestHttpMessages.java | 130 ++++ .../livy/client/common/TestSerializer.java | 78 +++ .../livy/client/common/TestTestUtils.java | 38 ++ client-http/pom.xml | 28 +- .../cloudera/livy/client/http/HttpClient.java | 193 ------ .../livy/client/http/HttpClientFactory.java | 40 -- .../com/cloudera/livy/client/http/HttpConf.java | 135 ---- .../livy/client/http/JobHandleImpl.java | 275 --------- .../livy/client/http/LivyConnection.java | 235 ------- .../org/apache/livy/client/http/HttpClient.java | 191 ++++++ .../livy/client/http/HttpClientFactory.java | 40 ++ .../org/apache/livy/client/http/HttpConf.java | 135 ++++ .../apache/livy/client/http/JobHandleImpl.java | 273 +++++++++ .../apache/livy/client/http/LivyConnection.java | 235 +++++++ .../com.cloudera.livy.LivyClientFactory | 1 - .../services/org.apache.livy.LivyClientFactory | 1 + .../livy/client/http/HttpClientSpec.scala | 290 --------- .../livy/client/http/LivyConnectionSpec.scala | 118 ---- .../livy/client/http/HttpClientSpec.scala | 290 +++++++++ .../livy/client/http/LivyConnectionSpec.scala | 118 ++++ core/pom.xml | 8 +- core/scala-2.10/pom.xml | 8 +- core/scala-2.11/pom.xml | 8 +- .../main/scala/com/cloudera/livy/Logging.scala | 54 -- .../main/scala/com/cloudera/livy/Utils.scala | 109 ---- .../src/main/scala/com/cloudera/livy/msgs.scala | 63 -- .../scala/com/cloudera/livy/sessions/Kind.scala | 68 --- .../cloudera/livy/sessions/SessionState.scala | 107 ---- .../main/scala/org/apache/livy/Logging.scala | 54 ++ core/src/main/scala/org/apache/livy/Utils.scala | 109 ++++ core/src/main/scala/org/apache/livy/msgs.scala | 63 ++ .../scala/org/apache/livy/sessions/Kind.scala | 68 +++ .../org/apache/livy/sessions/SessionState.scala | 107 ++++ .../cloudera/livy/LivyBaseUnitTestSuite.scala | 34 -- .../org/apache/livy/LivyBaseUnitTestSuite.scala | 34 ++ coverage/pom.xml | 6 +- examples/pom.xml | 14 +- .../java/com/cloudera/livy/examples/PiApp.java | 101 --- .../java/org/apache/livy/examples/PiApp.java | 101 +++ .../cloudera/livy/examples/WordCountApp.scala | 218 ------- .../org/apache/livy/examples/WordCountApp.scala | 218 +++++++ .../minicluster-dependencies/pom.xml | 6 +- .../minicluster-dependencies/scala-2.10/pom.xml | 8 +- .../minicluster-dependencies/scala-2.11/pom.xml | 8 +- integration-test/pom.xml | 6 +- .../framework/BaseIntegrationTestSuite.scala | 120 ---- .../cloudera/livy/test/framework/Cluster.scala | 160 ----- .../livy/test/framework/LivyRestClient.scala | 255 -------- .../livy/test/framework/MiniCluster.scala | 386 ------------ .../livy/test/framework/RealCluster.scala | 277 --------- .../framework/BaseIntegrationTestSuite.scala | 120 ++++ .../apache/livy/test/framework/Cluster.scala | 160 +++++ .../livy/test/framework/LivyRestClient.scala | 255 ++++++++ .../livy/test/framework/MiniCluster.scala | 386 ++++++++++++ .../livy/test/framework/RealCluster.scala | 277 +++++++++ .../scala/com/cloudera/livy/test/BatchIT.scala | 174 ------ .../com/cloudera/livy/test/InteractiveIT.scala | 205 ------- .../scala/com/cloudera/livy/test/JobApiIT.scala | 294 --------- .../scala/org/apache/livy/test/BatchIT.scala | 174 ++++++ .../org/apache/livy/test/InteractiveIT.scala | 205 +++++++ .../scala/org/apache/livy/test/JobApiIT.scala | 294 +++++++++ .../src/test/spark2/scala/Spark2JobApiIT.scala | 12 +- pom.xml | 4 +- python-api/pom.xml | 8 +- python-api/setup.py | 6 +- repl/pom.xml | 14 +- repl/scala-2.10/pom.xml | 8 +- .../cloudera/livy/repl/SparkInterpreter.scala | 178 ------ .../org/apache/livy/repl/SparkInterpreter.scala | 178 ++++++ .../livy/repl/SparkInterpreterSpec.scala | 86 --- .../apache/livy/repl/SparkInterpreterSpec.scala | 86 +++ repl/scala-2.11/pom.xml | 8 +- .../cloudera/livy/repl/SparkInterpreter.scala | 174 ------ .../org/apache/livy/repl/SparkInterpreter.scala | 174 ++++++ .../livy/repl/SparkInterpreterSpec.scala | 68 --- .../apache/livy/repl/SparkInterpreterSpec.scala | 68 +++ repl/src/main/resources/fake_shell.py | 2 +- .../livy/repl/AbstractSparkInterpreter.scala | 268 -------- .../cloudera/livy/repl/BypassPySparkJob.scala | 40 -- .../com/cloudera/livy/repl/Interpreter.scala | 54 -- .../cloudera/livy/repl/ProcessInterpreter.scala | 137 ----- .../livy/repl/PySparkJobProcessor.scala | 27 - .../cloudera/livy/repl/PythonInterpreter.scala | 293 --------- .../com/cloudera/livy/repl/ReplDriver.scala | 131 ---- .../scala/com/cloudera/livy/repl/Session.scala | 289 --------- .../livy/repl/SparkContextInitializer.scala | 124 ---- .../cloudera/livy/repl/SparkRInterpreter.scala | 324 ---------- .../scala/com/cloudera/livy/repl/package.scala | 29 - .../livy/repl/AbstractSparkInterpreter.scala | 268 ++++++++ .../org/apache/livy/repl/BypassPySparkJob.scala | 40 ++ .../org/apache/livy/repl/Interpreter.scala | 54 ++ .../apache/livy/repl/ProcessInterpreter.scala | 137 +++++ .../apache/livy/repl/PySparkJobProcessor.scala | 27 + .../apache/livy/repl/PythonInterpreter.scala | 293 +++++++++ .../scala/org/apache/livy/repl/ReplDriver.scala | 131 ++++ .../scala/org/apache/livy/repl/Session.scala | 289 +++++++++ .../livy/repl/SparkContextInitializer.scala | 124 ++++ .../apache/livy/repl/SparkRInterpreter.scala | 324 ++++++++++ .../scala/org/apache/livy/repl/package.scala | 29 + .../livy/repl/BaseInterpreterSpec.scala | 37 -- .../cloudera/livy/repl/BaseSessionSpec.scala | 87 --- .../livy/repl/PythonInterpreterSpec.scala | 284 --------- .../cloudera/livy/repl/PythonSessionSpec.scala | 206 ------- .../cloudera/livy/repl/ReplDriverSuite.scala | 68 --- .../livy/repl/ScalaInterpreterSpec.scala | 206 ------- .../com/cloudera/livy/repl/SessionSpec.scala | 126 ---- .../livy/repl/SparkRInterpreterSpec.scala | 107 ---- .../cloudera/livy/repl/SparkRSessionSpec.scala | 140 ----- .../cloudera/livy/repl/SparkSessionSpec.scala | 273 --------- .../apache/livy/repl/BaseInterpreterSpec.scala | 37 ++ .../org/apache/livy/repl/BaseSessionSpec.scala | 87 +++ .../livy/repl/PythonInterpreterSpec.scala | 284 +++++++++ .../apache/livy/repl/PythonSessionSpec.scala | 206 +++++++ .../org/apache/livy/repl/ReplDriverSuite.scala | 68 +++ .../apache/livy/repl/ScalaInterpreterSpec.scala | 206 +++++++ .../org/apache/livy/repl/SessionSpec.scala | 126 ++++ .../livy/repl/SparkRInterpreterSpec.scala | 107 ++++ .../apache/livy/repl/SparkRSessionSpec.scala | 140 +++++ .../org/apache/livy/repl/SparkSessionSpec.scala | 273 +++++++++ rsc/pom.xml | 18 +- .../com/cloudera/livy/rsc/BaseProtocol.java | 241 -------- .../com/cloudera/livy/rsc/BypassJobStatus.java | 40 -- .../java/com/cloudera/livy/rsc/ContextInfo.java | 37 -- .../com/cloudera/livy/rsc/ContextLauncher.java | 456 -------------- .../cloudera/livy/rsc/DriverProcessInfo.java | 42 -- .../com/cloudera/livy/rsc/FutureListener.java | 27 - .../com/cloudera/livy/rsc/JobHandleImpl.java | 106 ---- .../java/com/cloudera/livy/rsc/PingJob.java | 31 - .../java/com/cloudera/livy/rsc/RSCClient.java | 409 ------------- .../com/cloudera/livy/rsc/RSCClientFactory.java | 116 ---- .../java/com/cloudera/livy/rsc/RSCConf.java | 212 ------- .../com/cloudera/livy/rsc/ReplJobResults.java | 31 - .../main/java/com/cloudera/livy/rsc/Utils.java | 118 ---- .../cloudera/livy/rsc/driver/AddFileJob.java | 40 -- .../com/cloudera/livy/rsc/driver/AddJarJob.java | 42 -- .../com/cloudera/livy/rsc/driver/BypassJob.java | 51 -- .../livy/rsc/driver/BypassJobWrapper.java | 75 --- .../livy/rsc/driver/JobContextImpl.java | 147 ----- .../cloudera/livy/rsc/driver/JobWrapper.java | 99 --- .../livy/rsc/driver/MutableClassLoader.java | 34 -- .../com/cloudera/livy/rsc/driver/RSCDriver.java | 510 ---------------- .../livy/rsc/driver/RSCDriverBootstrapper.java | 89 --- .../com/cloudera/livy/rsc/driver/Statement.java | 59 -- .../livy/rsc/driver/StatementState.java | 86 --- .../cloudera/livy/rsc/rpc/KryoMessageCodec.java | 162 ----- .../java/com/cloudera/livy/rsc/rpc/README.md | 32 - .../java/com/cloudera/livy/rsc/rpc/Rpc.java | 460 -------------- .../cloudera/livy/rsc/rpc/RpcDispatcher.java | 219 ------- .../com/cloudera/livy/rsc/rpc/RpcException.java | 26 - .../com/cloudera/livy/rsc/rpc/RpcServer.java | 368 ----------- .../com/cloudera/livy/rsc/rpc/SaslHandler.java | 116 ---- .../java/org/apache/livy/rsc/BaseProtocol.java | 240 ++++++++ .../org/apache/livy/rsc/BypassJobStatus.java | 38 ++ .../java/org/apache/livy/rsc/ContextInfo.java | 37 ++ .../org/apache/livy/rsc/ContextLauncher.java | 457 ++++++++++++++ .../org/apache/livy/rsc/DriverProcessInfo.java | 42 ++ .../org/apache/livy/rsc/FutureListener.java | 27 + .../java/org/apache/livy/rsc/JobHandleImpl.java | 106 ++++ .../main/java/org/apache/livy/rsc/PingJob.java | 31 + .../java/org/apache/livy/rsc/RSCClient.java | 410 +++++++++++++ .../org/apache/livy/rsc/RSCClientFactory.java | 116 ++++ .../main/java/org/apache/livy/rsc/RSCConf.java | 212 +++++++ .../org/apache/livy/rsc/ReplJobResults.java | 31 + .../main/java/org/apache/livy/rsc/Utils.java | 118 ++++ .../org/apache/livy/rsc/driver/AddFileJob.java | 40 ++ .../org/apache/livy/rsc/driver/AddJarJob.java | 42 ++ .../org/apache/livy/rsc/driver/BypassJob.java | 51 ++ .../livy/rsc/driver/BypassJobWrapper.java | 75 +++ .../apache/livy/rsc/driver/JobContextImpl.java | 147 +++++ .../org/apache/livy/rsc/driver/JobWrapper.java | 99 +++ .../livy/rsc/driver/MutableClassLoader.java | 34 ++ .../org/apache/livy/rsc/driver/RSCDriver.java | 510 ++++++++++++++++ .../livy/rsc/driver/RSCDriverBootstrapper.java | 89 +++ .../org/apache/livy/rsc/driver/Statement.java | 59 ++ .../apache/livy/rsc/driver/StatementState.java | 86 +++ .../apache/livy/rsc/rpc/KryoMessageCodec.java | 162 +++++ .../main/java/org/apache/livy/rsc/rpc/README.md | 32 + .../main/java/org/apache/livy/rsc/rpc/Rpc.java | 460 ++++++++++++++ .../org/apache/livy/rsc/rpc/RpcDispatcher.java | 219 +++++++ .../org/apache/livy/rsc/rpc/RpcException.java | 26 + .../java/org/apache/livy/rsc/rpc/RpcServer.java | 368 +++++++++++ .../org/apache/livy/rsc/rpc/SaslHandler.java | 116 ++++ .../com.cloudera.livy.LivyClientFactory | 1 - .../services/org.apache.livy.LivyClientFactory | 1 + .../com/cloudera/livy/rsc/TestJobHandle.java | 98 --- .../com/cloudera/livy/rsc/TestSparkClient.java | 533 ---------------- .../livy/rsc/rpc/TestKryoMessageCodec.java | 232 ------- .../java/com/cloudera/livy/rsc/rpc/TestRpc.java | 337 ---------- .../java/org/apache/livy/rsc/TestJobHandle.java | 98 +++ .../org/apache/livy/rsc/TestSparkClient.java | 533 ++++++++++++++++ .../livy/rsc/rpc/TestKryoMessageCodec.java | 232 +++++++ .../java/org/apache/livy/rsc/rpc/TestRpc.java | 337 ++++++++++ scala-api/pom.xml | 14 +- scala-api/scala-2.10/pom.xml | 8 +- scala-api/scala-2.11/pom.xml | 8 +- .../livy/scalaapi/LivyScalaClient.scala | 165 ----- .../livy/scalaapi/ScalaJobContext.scala | 67 -- .../cloudera/livy/scalaapi/ScalaJobHandle.scala | 205 ------- .../com/cloudera/livy/scalaapi/package.scala | 50 -- .../apache/livy/scalaapi/LivyScalaClient.scala | 165 +++++ .../apache/livy/scalaapi/ScalaJobContext.scala | 67 ++ .../apache/livy/scalaapi/ScalaJobHandle.scala | 205 +++++++ .../org/apache/livy/scalaapi/package.scala | 50 ++ .../livy/scalaapi/ScalaClientTest.scala | 216 ------- .../livy/scalaapi/ScalaClientTestUtils.scala | 57 -- .../livy/scalaapi/ScalaJobHandleTest.scala | 190 ------ .../apache/livy/scalaapi/ScalaClientTest.scala | 216 +++++++ .../livy/scalaapi/ScalaClientTestUtils.scala | 57 ++ .../livy/scalaapi/ScalaJobHandleTest.scala | 190 ++++++ scala/pom.xml | 8 +- scalastyle.xml | 4 +- server/pom.xml | 12 +- .../livy/server/ui/static/all-sessions.js | 93 --- .../livy/server/ui/static/batches-table.html | 42 -- .../livy/server/ui/static/bootstrap.min.css | 14 - .../livy/server/ui/static/bootstrap.min.js | 7 - .../ui/static/dataTables.bootstrap.min.css | 1 - .../ui/static/dataTables.bootstrap.min.js | 8 - .../livy/server/ui/static/jquery-3.2.1.min.js | 4 - .../server/ui/static/jquery.dataTables.min.js | 167 ----- .../livy/server/ui/static/livy-mini-logo.png | Bin 1121 -> 0 bytes .../cloudera/livy/server/ui/static/livy-ui.css | 20 - .../livy/server/ui/static/sessions-table.html | 59 -- .../livy/server/ui/static/all-sessions.js | 93 +++ .../livy/server/ui/static/batches-table.html | 42 ++ .../livy/server/ui/static/bootstrap.min.css | 14 + .../livy/server/ui/static/bootstrap.min.js | 7 + .../ui/static/dataTables.bootstrap.min.css | 1 + .../ui/static/dataTables.bootstrap.min.js | 8 + .../livy/server/ui/static/jquery-3.2.1.min.js | 4 + .../server/ui/static/jquery.dataTables.min.js | 167 +++++ .../livy/server/ui/static/livy-mini-logo.png | Bin 0 -> 1121 bytes .../apache/livy/server/ui/static/livy-ui.css | 20 + .../livy/server/ui/static/sessions-table.html | 59 ++ .../main/scala/com/cloudera/livy/LivyConf.scala | 297 --------- .../main/scala/com/cloudera/livy/package.scala | 73 --- .../com/cloudera/livy/server/AccessFilter.scala | 45 -- .../livy/server/ApiVersioningSupport.scala | 93 --- .../com/cloudera/livy/server/ApiVersions.scala | 35 -- .../com/cloudera/livy/server/CsrfFilter.scala | 49 -- .../com/cloudera/livy/server/JsonServlet.scala | 143 ----- .../com/cloudera/livy/server/LivyServer.scala | 348 ----------- .../cloudera/livy/server/SessionServlet.scala | 215 ------- .../com/cloudera/livy/server/WebServer.scala | 113 ---- .../livy/server/batch/BatchSession.scala | 173 ------ .../livy/server/batch/BatchSessionServlet.scala | 67 -- .../livy/server/batch/CreateBatchRequest.scala | 56 -- .../interactive/CreateInteractiveRequest.scala | 56 -- .../server/interactive/InteractiveSession.scala | 609 ------------------- .../interactive/InteractiveSessionServlet.scala | 247 -------- .../server/interactive/SessionHeartbeat.scala | 113 ---- .../server/recovery/BlackholeStateStore.scala | 36 -- .../server/recovery/FileSystemStateStore.scala | 132 ---- .../livy/server/recovery/SessionStore.scala | 96 --- .../livy/server/recovery/StateStore.scala | 111 ---- .../server/recovery/ZooKeeperStateStore.scala | 118 ---- .../com/cloudera/livy/server/ui/UIServlet.scala | 78 --- .../com/cloudera/livy/sessions/Session.scala | 264 -------- .../cloudera/livy/sessions/SessionManager.scala | 188 ------ .../scala/com/cloudera/livy/utils/Clock.scala | 38 -- .../livy/utils/LineBufferedProcess.scala | 51 -- .../livy/utils/LineBufferedStream.scala | 97 --- .../cloudera/livy/utils/LivySparkUtils.scala | 197 ------ .../com/cloudera/livy/utils/SparkApp.scala | 106 ---- .../com/cloudera/livy/utils/SparkProcApp.scala | 60 -- .../livy/utils/SparkProcessBuilder.scala | 219 ------- .../com/cloudera/livy/utils/SparkYarnApp.scala | 312 ---------- .../main/scala/org/apache/livy/LivyConf.scala | 297 +++++++++ .../main/scala/org/apache/livy/package.scala | 73 +++ .../org/apache/livy/server/AccessFilter.scala | 45 ++ .../livy/server/ApiVersioningSupport.scala | 93 +++ .../org/apache/livy/server/ApiVersions.scala | 35 ++ .../org/apache/livy/server/CsrfFilter.scala | 48 ++ .../org/apache/livy/server/JsonServlet.scala | 143 +++++ .../org/apache/livy/server/LivyServer.scala | 348 +++++++++++ .../org/apache/livy/server/SessionServlet.scala | 215 +++++++ .../org/apache/livy/server/WebServer.scala | 113 ++++ .../apache/livy/server/batch/BatchSession.scala | 173 ++++++ .../livy/server/batch/BatchSessionServlet.scala | 67 ++ .../livy/server/batch/CreateBatchRequest.scala | 56 ++ .../interactive/CreateInteractiveRequest.scala | 56 ++ .../server/interactive/InteractiveSession.scala | 608 ++++++++++++++++++ .../interactive/InteractiveSessionServlet.scala | 247 ++++++++ .../server/interactive/SessionHeartbeat.scala | 113 ++++ .../server/recovery/BlackholeStateStore.scala | 36 ++ .../server/recovery/FileSystemStateStore.scala | 132 ++++ .../livy/server/recovery/SessionStore.scala | 96 +++ .../livy/server/recovery/StateStore.scala | 111 ++++ .../server/recovery/ZooKeeperStateStore.scala | 118 ++++ .../org/apache/livy/server/ui/UIServlet.scala | 76 +++ .../org/apache/livy/sessions/Session.scala | 264 ++++++++ .../apache/livy/sessions/SessionManager.scala | 188 ++++++ .../scala/org/apache/livy/utils/Clock.scala | 38 ++ .../apache/livy/utils/LineBufferedProcess.scala | 51 ++ .../apache/livy/utils/LineBufferedStream.scala | 97 +++ .../org/apache/livy/utils/LivySparkUtils.scala | 196 ++++++ .../scala/org/apache/livy/utils/SparkApp.scala | 105 ++++ .../org/apache/livy/utils/SparkProcApp.scala | 59 ++ .../apache/livy/utils/SparkProcessBuilder.scala | 218 +++++++ .../org/apache/livy/utils/SparkYarnApp.scala | 311 ++++++++++ .../livy/server/ApiVersioningSupportSpec.scala | 124 ---- .../livy/server/BaseJsonServletSpec.scala | 141 ----- .../livy/server/BaseSessionServletSpec.scala | 82 --- .../cloudera/livy/server/JsonServletSpec.scala | 149 ----- .../livy/server/SessionServletSpec.scala | 156 ----- .../livy/server/batch/BatchServletSpec.scala | 149 ----- .../livy/server/batch/BatchSessionSpec.scala | 113 ---- .../server/batch/CreateBatchRequestSpec.scala | 55 -- .../BaseInteractiveServletSpec.scala | 74 --- .../CreateInteractiveRequestSpec.scala | 55 -- .../InteractiveSessionServletSpec.scala | 183 ------ .../interactive/InteractiveSessionSpec.scala | 264 -------- .../livy/server/interactive/JobApiSpec.scala | 227 ------- .../interactive/SessionHeartbeatSpec.scala | 87 --- .../recovery/BlackholeStateStoreSpec.scala | 47 -- .../recovery/FileSystemStateStoreSpec.scala | 192 ------ .../livy/server/recovery/SessionStoreSpec.scala | 108 ---- .../livy/server/recovery/StateStoreSpec.scala | 65 -- .../recovery/ZooKeeperStateStoreSpec.scala | 174 ------ .../cloudera/livy/sessions/MockSession.scala | 34 -- .../livy/sessions/SessionManagerSpec.scala | 205 ------- .../cloudera/livy/sessions/SessionSpec.scala | 98 --- .../livy/utils/LivySparkUtilsSuite.scala | 140 ----- .../cloudera/livy/utils/SparkYarnAppSpec.scala | 353 ----------- .../livy/server/ApiVersioningSupportSpec.scala | 124 ++++ .../livy/server/BaseJsonServletSpec.scala | 141 +++++ .../livy/server/BaseSessionServletSpec.scala | 82 +++ .../apache/livy/server/JsonServletSpec.scala | 149 +++++ .../apache/livy/server/SessionServletSpec.scala | 155 +++++ .../livy/server/batch/BatchServletSpec.scala | 149 +++++ .../livy/server/batch/BatchSessionSpec.scala | 113 ++++ .../server/batch/CreateBatchRequestSpec.scala | 55 ++ .../BaseInteractiveServletSpec.scala | 74 +++ .../CreateInteractiveRequestSpec.scala | 55 ++ .../InteractiveSessionServletSpec.scala | 183 ++++++ .../interactive/InteractiveSessionSpec.scala | 264 ++++++++ .../livy/server/interactive/JobApiSpec.scala | 227 +++++++ .../interactive/SessionHeartbeatSpec.scala | 87 +++ .../recovery/BlackholeStateStoreSpec.scala | 47 ++ .../recovery/FileSystemStateStoreSpec.scala | 192 ++++++ .../livy/server/recovery/SessionStoreSpec.scala | 108 ++++ .../livy/server/recovery/StateStoreSpec.scala | 65 ++ .../recovery/ZooKeeperStateStoreSpec.scala | 174 ++++++ .../org/apache/livy/sessions/MockSession.scala | 34 ++ .../livy/sessions/SessionManagerSpec.scala | 205 +++++++ .../org/apache/livy/sessions/SessionSpec.scala | 98 +++ .../apache/livy/utils/LivySparkUtilsSuite.scala | 140 +++++ .../apache/livy/utils/SparkYarnAppSpec.scala | 352 +++++++++++ test-lib/pom.xml | 10 +- .../com/cloudera/livy/test/apps/FailingApp.java | 39 -- .../cloudera/livy/test/apps/SimpleSparkApp.java | 71 --- .../java/com/cloudera/livy/test/jobs/Echo.java | 36 -- .../com/cloudera/livy/test/jobs/Failure.java | 34 -- .../com/cloudera/livy/test/jobs/FileReader.java | 79 --- .../cloudera/livy/test/jobs/GetCurrentUser.java | 32 - .../cloudera/livy/test/jobs/SQLGetTweets.java | 76 --- .../com/cloudera/livy/test/jobs/Sleeper.java | 37 -- .../com/cloudera/livy/test/jobs/SmallCount.java | 48 -- .../com/cloudera/livy/test/jobs/VoidJob.java | 28 - .../org/apache/livy/test/apps/FailingApp.java | 39 ++ .../apache/livy/test/apps/SimpleSparkApp.java | 71 +++ .../java/org/apache/livy/test/jobs/Echo.java | 36 ++ .../java/org/apache/livy/test/jobs/Failure.java | 34 ++ .../org/apache/livy/test/jobs/FileReader.java | 79 +++ .../apache/livy/test/jobs/GetCurrentUser.java | 32 + .../org/apache/livy/test/jobs/SQLGetTweets.java | 76 +++ .../java/org/apache/livy/test/jobs/Sleeper.java | 37 ++ .../org/apache/livy/test/jobs/SmallCount.java | 48 ++ .../java/org/apache/livy/test/jobs/VoidJob.java | 28 + .../com/cloudera/livy/test/jobs/ScalaEcho.scala | 32 - .../org/apache/livy/test/jobs/ScalaEcho.scala | 32 + .../livy/test/jobs/spark2/DatasetTest.java | 63 -- .../livy/test/jobs/spark2/SparkSessionTest.java | 38 -- .../livy/test/jobs/spark2/DatasetTest.java | 63 ++ .../livy/test/jobs/spark2/SparkSessionTest.java | 38 ++ 422 files changed, 25580 insertions(+), 25596 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/README.rst ---------------------------------------------------------------------- diff --git a/README.rst b/README.rst index d72c955..9f14fa0 100644 --- a/README.rst +++ b/README.rst @@ -190,7 +190,7 @@ And add the Livy client dependency: To be able to compile code that uses Spark APIs, also add the correspondent Spark dependencies. -To run Spark jobs within your applications, extend ``com.cloudera.livy.Job`` and implement +To run Spark jobs within your applications, extend ``org.apache.livy.Job`` and implement the functionality you need. Here's an example job that calculates an approximate value for Pi: .. code:: java http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/pom.xml ---------------------------------------------------------------------- diff --git a/api/pom.xml b/api/pom.xml index 156d987..0b1411e 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -18,14 +18,14 @@ <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> - <groupId>com.cloudera.livy</groupId> + <groupId>org.apache.livy</groupId> <artifactId>livy-main</artifactId> - <version>0.4.0-SNAPSHOT</version> + <version>0.4.0-incubating-SNAPSHOT</version> </parent> - <groupId>com.cloudera.livy</groupId> + <groupId>org.apache.livy</groupId> <artifactId>livy-api</artifactId> - <version>0.4.0-SNAPSHOT</version> + <version>0.4.0-incubating-SNAPSHOT</version> <packaging>jar</packaging> <properties> http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/Job.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/com/cloudera/livy/Job.java b/api/src/main/java/com/cloudera/livy/Job.java deleted file mode 100644 index fc96b72..0000000 --- a/api/src/main/java/com/cloudera/livy/Job.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.livy; - -import java.io.Serializable; - -/** - * Interface for a Spark remote job. - */ -public interface Job<T> extends Serializable { - - T call(JobContext jc) throws Exception; - -} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/JobContext.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/com/cloudera/livy/JobContext.java b/api/src/main/java/com/cloudera/livy/JobContext.java deleted file mode 100644 index d32adef..0000000 --- a/api/src/main/java/com/cloudera/livy/JobContext.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.livy; - -import java.io.File; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.sql.SQLContext; -import org.apache.spark.sql.hive.HiveContext; -import org.apache.spark.streaming.api.java.JavaStreamingContext; - -/** - * Holds runtime information about the job execution context. - * - * An instance of this class is kept on the node hosting a remote Spark context and is made - * available to jobs being executed via RemoteSparkContext#submit(). - */ -public interface JobContext { - - /** The shared SparkContext instance. */ - JavaSparkContext sc(); - - /** The shared SQLContext instance. */ - SQLContext sqlctx(); - - /** The shared HiveContext instance. */ - HiveContext hivectx(); - - /** Returns the JavaStreamingContext which has already been created. */ - JavaStreamingContext streamingctx(); - - /** - * Creates the SparkStreaming context. - * - * @param batchDuration Time interval at which streaming data will be divided into batches, - * in milliseconds. - */ - void createStreamingContext(long batchDuration); - - /** Stops the SparkStreaming context. */ - void stopStreamingCtx(); - - /** - * Returns a local tmp dir specific to the context - */ - File getLocalTmpDir(); - - /** - * Returns SparkSession if it existed, otherwise throws Exception. - */ - <E> E sparkSession() throws Exception; -} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/JobHandle.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/com/cloudera/livy/JobHandle.java b/api/src/main/java/com/cloudera/livy/JobHandle.java deleted file mode 100644 index 1da606e..0000000 --- a/api/src/main/java/com/cloudera/livy/JobHandle.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.livy; - -import java.util.List; -import java.util.concurrent.Future; - -/** - * A handle to a submitted job. Allows for monitoring and controlling of the running remote job. - */ -public interface JobHandle<T> extends Future<T> { - - /** - * Return the current state of the job. - */ - State getState(); - - /** - * Add a listener to the job handle. If the job's state is not SENT, a callback for the - * corresponding state will be invoked immediately. - * - * @param l The listener to add. - */ - void addListener(Listener<T> l); - - /** - * The current state of the submitted job. - */ - static enum State { - SENT, - QUEUED, - STARTED, - CANCELLED, - FAILED, - SUCCEEDED; - } - - /** - * A listener for monitoring the state of the job in the remote context. Callbacks are called - * when the corresponding state change occurs. - */ - static interface Listener<T> { - - /** - * Notifies when a job has been queued for execution on the remote context. Note that it is - * possible for jobs to bypass this state and got directly from the SENT state to the STARTED - * state. - */ - void onJobQueued(JobHandle<T> job); - - void onJobStarted(JobHandle<T> job); - - void onJobCancelled(JobHandle<T> job); - - void onJobFailed(JobHandle<T> job, Throwable cause); - - void onJobSucceeded(JobHandle<T> job, T result); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/LivyClient.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/com/cloudera/livy/LivyClient.java b/api/src/main/java/com/cloudera/livy/LivyClient.java deleted file mode 100644 index 73e318f..0000000 --- a/api/src/main/java/com/cloudera/livy/LivyClient.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.livy; - -import java.io.File; -import java.net.URI; -import java.util.concurrent.Future; - -/** - * A client for submitting Spark-based jobs to a Livy backend. - */ -public interface LivyClient { - - /** - * Submits a job for asynchronous execution. - * - * @param job The job to execute. - * @return A handle that be used to monitor the job. - */ - <T> JobHandle<T> submit(Job<T> job); - - /** - * Asks the remote context to run a job immediately. - * <p/> - * Normally, the remote context will queue jobs and execute them based on how many worker - * threads have been configured. This method will run the submitted job in the same thread - * processing the RPC message, so that queueing does not apply. - * <p/> - * It's recommended that this method only be used to run code that finishes quickly. This - * avoids interfering with the normal operation of the context. - * <p/> - * Note: the {@link JobContext#monitor()} functionality is not available when using this method. - * - * @param job The job to execute. - * @return A future to monitor the result of the job. - */ - <T> Future<T> run(Job<T> job); - - /** - * Stops the remote context. - * - * Any pending jobs will be cancelled, and the remote context will be torn down. - * - * @param shutdownContext Whether to shutdown the underlying Spark context. If false, the - * context will keep running and it's still possible to send commands - * to it, if the backend being used supports it. - */ - void stop(boolean shutdownContext); - - /** - * Upload a jar to be added to the Spark application classpath - * @param jar The local file to be uploaded - * @return A future that can be used to monitor this operation - */ - Future<?> uploadJar(File jar); - - /** - * Adds a jar file to the running remote context. - * <p> - * Note that the URL should be reachable by the Spark driver process. If running the driver - * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist - * on that node (and not on the client machine). - * <p> - * If the provided URI has no scheme, it's considered to be relative to the default file system - * configured in the Livy server. - * - * @param uri The location of the jar file. - * @return A future that can be used to monitor the operation. - */ - Future<?> addJar(URI uri); - - /** - * Upload a file to be passed to the Spark application - * @param file The local file to be uploaded - * @return A future that can be used to monitor this operation - */ - Future<?> uploadFile(File file); - - /** - * Adds a file to the running remote context. - * <p> - * Note that the URL should be reachable by the Spark driver process. If running the driver - * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist - * on that node (and not on the client machine). - * <p> - * If the provided URI has no scheme, it's considered to be relative to the default file system - * configured in the Livy server. - * - * @param uri The location of the file. - * @return A future that can be used to monitor the operation. - */ - Future<?> addFile(URI uri); - -} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/LivyClientBuilder.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/com/cloudera/livy/LivyClientBuilder.java b/api/src/main/java/com/cloudera/livy/LivyClientBuilder.java deleted file mode 100644 index 5203f04..0000000 --- a/api/src/main/java/com/cloudera/livy/LivyClientBuilder.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.livy; - -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Map; -import java.util.Properties; -import java.util.ServiceLoader; -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * A builder for Livy clients. - */ -public final class LivyClientBuilder { - - public static final String LIVY_URI_KEY = "livy.uri"; - - private final Properties config; - - /** - * Creates a new builder that will automatically load the default Livy and Spark configuration - * from the classpath. - */ - public LivyClientBuilder() throws IOException { - this(true); - } - - /** - * Creates a new builder that will optionally load the default Livy and Spark configuration - * from the classpath. - * - * Livy client configuration is stored in a file called "livy-client.conf", and Spark client - * configuration is stored in a file called "spark-defaults.conf", both in the root of the - * application's classpath. Livy configuration takes precedence over Spark's (in case - * configuration entries are duplicated), and configuration set in this builder object will - * override the values in those files. - */ - public LivyClientBuilder(boolean loadDefaults) throws IOException { - this.config = new Properties(); - - if (loadDefaults) { - String[] confFiles = { "spark-defaults.conf", "livy-client.conf" }; - - for (String file : confFiles) { - URL url = classLoader().getResource(file); - if (url != null) { - Reader r = new InputStreamReader(url.openStream(), UTF_8); - try { - config.load(r); - } finally { - r.close(); - } - } - } - } - } - - public LivyClientBuilder setURI(URI uri) { - config.setProperty(LIVY_URI_KEY, uri.toString()); - return this; - } - - public LivyClientBuilder setConf(String key, String value) { - if (value != null) { - config.setProperty(key, value); - } else { - config.remove(key); - } - return this; - } - - public LivyClientBuilder setAll(Map<String, String> props) { - config.putAll(props); - return this; - } - - public LivyClientBuilder setAll(Properties props) { - config.putAll(props); - return this; - } - - public LivyClient build() { - String uriStr = config.getProperty(LIVY_URI_KEY); - if (uriStr == null) { - throw new IllegalArgumentException("URI must be provided."); - } - URI uri; - try { - uri = new URI(uriStr); - } catch (URISyntaxException e) { - throw new IllegalArgumentException("Invalid URI.", e); - } - - LivyClient client = null; - ServiceLoader<LivyClientFactory> loader = ServiceLoader.load(LivyClientFactory.class, - classLoader()); - if (!loader.iterator().hasNext()) { - throw new IllegalStateException("No LivyClientFactory implementation was found."); - } - - Exception error = null; - for (LivyClientFactory factory : loader) { - try { - client = factory.createClient(uri, config); - } catch (Exception e) { - if (!(e instanceof RuntimeException)) { - e = new RuntimeException(e); - } - throw (RuntimeException) e; - } - if (client != null) { - break; - } - } - - if (client == null) { - // Redact any user information from the URI when throwing user-visible exceptions that might - // be logged. - if (uri.getUserInfo() != null) { - try { - uri = new URI(uri.getScheme(), "[redacted]", uri.getHost(), uri.getPort(), uri.getPath(), - uri.getQuery(), uri.getFragment()); - } catch (URISyntaxException e) { - // Shouldn't really happen. - throw new RuntimeException(e); - } - } - - throw new IllegalArgumentException(String.format( - "URI '%s' is not supported by any registered client factories.", uri)); - } - return client; - } - - private ClassLoader classLoader() { - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - if (cl == null) { - cl = getClass().getClassLoader(); - } - return cl; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/LivyClientFactory.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/com/cloudera/livy/LivyClientFactory.java b/api/src/main/java/com/cloudera/livy/LivyClientFactory.java deleted file mode 100644 index 095d5e4..0000000 --- a/api/src/main/java/com/cloudera/livy/LivyClientFactory.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.livy; - -import java.net.URI; -import java.util.Properties; - -import com.cloudera.livy.annotations.Private; - -/** - * A factory for Livy clients. Client implementations can register themselves by using the - * Java services mechanism, providing implementations of this interface. - * <p> - * Client applications do not need to use this interface directly. Instead, use - * {@link LivyClientBuilder}. - * - * @see java.util.ServiceLoader - */ -@Private -public interface LivyClientFactory { - - /** - * Instantiates a new client if the given URI is supported by the implementation. - * - * @param uri URI pointing at the livy backend to use. - */ - LivyClient createClient(URI uri, Properties config); - -} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/com/cloudera/livy/annotations/Private.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/com/cloudera/livy/annotations/Private.java b/api/src/main/java/com/cloudera/livy/annotations/Private.java deleted file mode 100644 index 4fc62f7..0000000 --- a/api/src/main/java/com/cloudera/livy/annotations/Private.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.livy.annotations; - -import java.lang.annotation.Documented; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; - -/** - * Indicates an API that is considered private to Livy and should not be used by client - * applications. - */ -@Documented -@Retention(RetentionPolicy.CLASS) -public @interface Private { } http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/Job.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/org/apache/livy/Job.java b/api/src/main/java/org/apache/livy/Job.java new file mode 100644 index 0000000..cbb6b9a --- /dev/null +++ b/api/src/main/java/org/apache/livy/Job.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.livy; + +import java.io.Serializable; + +/** + * Interface for a Spark remote job. + */ +public interface Job<T> extends Serializable { + + T call(JobContext jc) throws Exception; + +} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/JobContext.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/org/apache/livy/JobContext.java b/api/src/main/java/org/apache/livy/JobContext.java new file mode 100644 index 0000000..47027b7 --- /dev/null +++ b/api/src/main/java/org/apache/livy/JobContext.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.livy; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.SQLContext; +import org.apache.spark.sql.hive.HiveContext; +import org.apache.spark.streaming.api.java.JavaStreamingContext; + +/** + * Holds runtime information about the job execution context. + * + * An instance of this class is kept on the node hosting a remote Spark context and is made + * available to jobs being executed via RemoteSparkContext#submit(). + */ +public interface JobContext { + + /** The shared SparkContext instance. */ + JavaSparkContext sc(); + + /** The shared SQLContext instance. */ + SQLContext sqlctx(); + + /** The shared HiveContext instance. */ + HiveContext hivectx(); + + /** Returns the JavaStreamingContext which has already been created. */ + JavaStreamingContext streamingctx(); + + /** + * Creates the SparkStreaming context. + * + * @param batchDuration Time interval at which streaming data will be divided into batches, + * in milliseconds. + */ + void createStreamingContext(long batchDuration); + + /** Stops the SparkStreaming context. */ + void stopStreamingCtx(); + + /** + * Returns a local tmp dir specific to the context + */ + File getLocalTmpDir(); + + /** + * Returns SparkSession if it existed, otherwise throws Exception. + */ + <E> E sparkSession() throws Exception; +} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/JobHandle.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/org/apache/livy/JobHandle.java b/api/src/main/java/org/apache/livy/JobHandle.java new file mode 100644 index 0000000..d2a71ff --- /dev/null +++ b/api/src/main/java/org/apache/livy/JobHandle.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.livy; + +import java.util.List; +import java.util.concurrent.Future; + +/** + * A handle to a submitted job. Allows for monitoring and controlling of the running remote job. + */ +public interface JobHandle<T> extends Future<T> { + + /** + * Return the current state of the job. + */ + State getState(); + + /** + * Add a listener to the job handle. If the job's state is not SENT, a callback for the + * corresponding state will be invoked immediately. + * + * @param l The listener to add. + */ + void addListener(Listener<T> l); + + /** + * The current state of the submitted job. + */ + static enum State { + SENT, + QUEUED, + STARTED, + CANCELLED, + FAILED, + SUCCEEDED; + } + + /** + * A listener for monitoring the state of the job in the remote context. Callbacks are called + * when the corresponding state change occurs. + */ + static interface Listener<T> { + + /** + * Notifies when a job has been queued for execution on the remote context. Note that it is + * possible for jobs to bypass this state and got directly from the SENT state to the STARTED + * state. + */ + void onJobQueued(JobHandle<T> job); + + void onJobStarted(JobHandle<T> job); + + void onJobCancelled(JobHandle<T> job); + + void onJobFailed(JobHandle<T> job, Throwable cause); + + void onJobSucceeded(JobHandle<T> job, T result); + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/LivyClient.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/org/apache/livy/LivyClient.java b/api/src/main/java/org/apache/livy/LivyClient.java new file mode 100644 index 0000000..72408e4 --- /dev/null +++ b/api/src/main/java/org/apache/livy/LivyClient.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.livy; + +import java.io.File; +import java.net.URI; +import java.util.concurrent.Future; + +/** + * A client for submitting Spark-based jobs to a Livy backend. + */ +public interface LivyClient { + + /** + * Submits a job for asynchronous execution. + * + * @param job The job to execute. + * @return A handle that be used to monitor the job. + */ + <T> JobHandle<T> submit(Job<T> job); + + /** + * Asks the remote context to run a job immediately. + * <p/> + * Normally, the remote context will queue jobs and execute them based on how many worker + * threads have been configured. This method will run the submitted job in the same thread + * processing the RPC message, so that queueing does not apply. + * <p/> + * It's recommended that this method only be used to run code that finishes quickly. This + * avoids interfering with the normal operation of the context. + * <p/> + * Note: the {@link JobContext#monitor()} functionality is not available when using this method. + * + * @param job The job to execute. + * @return A future to monitor the result of the job. + */ + <T> Future<T> run(Job<T> job); + + /** + * Stops the remote context. + * + * Any pending jobs will be cancelled, and the remote context will be torn down. + * + * @param shutdownContext Whether to shutdown the underlying Spark context. If false, the + * context will keep running and it's still possible to send commands + * to it, if the backend being used supports it. + */ + void stop(boolean shutdownContext); + + /** + * Upload a jar to be added to the Spark application classpath + * @param jar The local file to be uploaded + * @return A future that can be used to monitor this operation + */ + Future<?> uploadJar(File jar); + + /** + * Adds a jar file to the running remote context. + * <p> + * Note that the URL should be reachable by the Spark driver process. If running the driver + * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist + * on that node (and not on the client machine). + * <p> + * If the provided URI has no scheme, it's considered to be relative to the default file system + * configured in the Livy server. + * + * @param uri The location of the jar file. + * @return A future that can be used to monitor the operation. + */ + Future<?> addJar(URI uri); + + /** + * Upload a file to be passed to the Spark application + * @param file The local file to be uploaded + * @return A future that can be used to monitor this operation + */ + Future<?> uploadFile(File file); + + /** + * Adds a file to the running remote context. + * <p> + * Note that the URL should be reachable by the Spark driver process. If running the driver + * in cluster mode, it may reside on a different host, meaning "file:" URLs have to exist + * on that node (and not on the client machine). + * <p> + * If the provided URI has no scheme, it's considered to be relative to the default file system + * configured in the Livy server. + * + * @param uri The location of the file. + * @return A future that can be used to monitor the operation. + */ + Future<?> addFile(URI uri); + +} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/LivyClientBuilder.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/org/apache/livy/LivyClientBuilder.java b/api/src/main/java/org/apache/livy/LivyClientBuilder.java new file mode 100644 index 0000000..1acc1f8 --- /dev/null +++ b/api/src/main/java/org/apache/livy/LivyClientBuilder.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.livy; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Map; +import java.util.Properties; +import java.util.ServiceLoader; +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * A builder for Livy clients. + */ +public final class LivyClientBuilder { + + public static final String LIVY_URI_KEY = "livy.uri"; + + private final Properties config; + + /** + * Creates a new builder that will automatically load the default Livy and Spark configuration + * from the classpath. + */ + public LivyClientBuilder() throws IOException { + this(true); + } + + /** + * Creates a new builder that will optionally load the default Livy and Spark configuration + * from the classpath. + * + * Livy client configuration is stored in a file called "livy-client.conf", and Spark client + * configuration is stored in a file called "spark-defaults.conf", both in the root of the + * application's classpath. Livy configuration takes precedence over Spark's (in case + * configuration entries are duplicated), and configuration set in this builder object will + * override the values in those files. + */ + public LivyClientBuilder(boolean loadDefaults) throws IOException { + this.config = new Properties(); + + if (loadDefaults) { + String[] confFiles = { "spark-defaults.conf", "livy-client.conf" }; + + for (String file : confFiles) { + URL url = classLoader().getResource(file); + if (url != null) { + Reader r = new InputStreamReader(url.openStream(), UTF_8); + try { + config.load(r); + } finally { + r.close(); + } + } + } + } + } + + public LivyClientBuilder setURI(URI uri) { + config.setProperty(LIVY_URI_KEY, uri.toString()); + return this; + } + + public LivyClientBuilder setConf(String key, String value) { + if (value != null) { + config.setProperty(key, value); + } else { + config.remove(key); + } + return this; + } + + public LivyClientBuilder setAll(Map<String, String> props) { + config.putAll(props); + return this; + } + + public LivyClientBuilder setAll(Properties props) { + config.putAll(props); + return this; + } + + public LivyClient build() { + String uriStr = config.getProperty(LIVY_URI_KEY); + if (uriStr == null) { + throw new IllegalArgumentException("URI must be provided."); + } + URI uri; + try { + uri = new URI(uriStr); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid URI.", e); + } + + LivyClient client = null; + ServiceLoader<LivyClientFactory> loader = ServiceLoader.load(LivyClientFactory.class, + classLoader()); + if (!loader.iterator().hasNext()) { + throw new IllegalStateException("No LivyClientFactory implementation was found."); + } + + Exception error = null; + for (LivyClientFactory factory : loader) { + try { + client = factory.createClient(uri, config); + } catch (Exception e) { + if (!(e instanceof RuntimeException)) { + e = new RuntimeException(e); + } + throw (RuntimeException) e; + } + if (client != null) { + break; + } + } + + if (client == null) { + // Redact any user information from the URI when throwing user-visible exceptions that might + // be logged. + if (uri.getUserInfo() != null) { + try { + uri = new URI(uri.getScheme(), "[redacted]", uri.getHost(), uri.getPort(), uri.getPath(), + uri.getQuery(), uri.getFragment()); + } catch (URISyntaxException e) { + // Shouldn't really happen. + throw new RuntimeException(e); + } + } + + throw new IllegalArgumentException(String.format( + "URI '%s' is not supported by any registered client factories.", uri)); + } + return client; + } + + private ClassLoader classLoader() { + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + if (cl == null) { + cl = getClass().getClassLoader(); + } + return cl; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/LivyClientFactory.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/org/apache/livy/LivyClientFactory.java b/api/src/main/java/org/apache/livy/LivyClientFactory.java new file mode 100644 index 0000000..2c51d07 --- /dev/null +++ b/api/src/main/java/org/apache/livy/LivyClientFactory.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.livy; + +import java.net.URI; +import java.util.Properties; + +import org.apache.livy.annotations.Private; + +/** + * A factory for Livy clients. Client implementations can register themselves by using the + * Java services mechanism, providing implementations of this interface. + * <p> + * Client applications do not need to use this interface directly. Instead, use + * {@link LivyClientBuilder}. + * + * @see java.util.ServiceLoader + */ +@Private +public interface LivyClientFactory { + + /** + * Instantiates a new client if the given URI is supported by the implementation. + * + * @param uri URI pointing at the livy backend to use. + */ + LivyClient createClient(URI uri, Properties config); + +} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/main/java/org/apache/livy/annotations/Private.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/org/apache/livy/annotations/Private.java b/api/src/main/java/org/apache/livy/annotations/Private.java new file mode 100644 index 0000000..546103f --- /dev/null +++ b/api/src/main/java/org/apache/livy/annotations/Private.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.livy.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Indicates an API that is considered private to Livy and should not be used by client + * applications. + */ +@Documented +@Retention(RetentionPolicy.CLASS) +public @interface Private { } http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/test/java/com/cloudera/livy/TestClientFactory.java ---------------------------------------------------------------------- diff --git a/api/src/test/java/com/cloudera/livy/TestClientFactory.java b/api/src/test/java/com/cloudera/livy/TestClientFactory.java deleted file mode 100644 index c1ff7ac..0000000 --- a/api/src/test/java/com/cloudera/livy/TestClientFactory.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.livy; - -import java.io.File; -import java.net.URI; -import java.util.Properties; -import java.util.concurrent.Future; - -public class TestClientFactory implements LivyClientFactory { - - @Override - public LivyClient createClient(URI uri, Properties config) { - switch (uri.getPath()) { - case "match": - return new Client(config); - - case "error": - throw new IllegalStateException("error"); - - default: - return null; - } - } - - public static class Client implements LivyClient { - - public final Properties config; - - private Client(Properties config) { - this.config = config; - } - - @Override - public <T> JobHandle<T> submit(Job<T> job) { - throw new UnsupportedOperationException(); - } - - @Override - public <T> Future<T> run(Job<T> job) { - throw new UnsupportedOperationException(); - } - - @Override - public void stop(boolean shutdownContext) { - throw new UnsupportedOperationException(); - } - - @Override - public Future<?> uploadJar(File jar) { - throw new UnsupportedOperationException(); - } - - @Override - public Future<?> addJar(URI uri) { - throw new UnsupportedOperationException(); - } - - @Override - public Future<?> uploadFile(File file) { - throw new UnsupportedOperationException(); - } - - @Override - public Future<?> addFile(URI uri) { - throw new UnsupportedOperationException(); - } - -} - -} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/test/java/com/cloudera/livy/TestLivyClientBuilder.java ---------------------------------------------------------------------- diff --git a/api/src/test/java/com/cloudera/livy/TestLivyClientBuilder.java b/api/src/test/java/com/cloudera/livy/TestLivyClientBuilder.java deleted file mode 100644 index 44144ed..0000000 --- a/api/src/test/java/com/cloudera/livy/TestLivyClientBuilder.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.livy; - -import java.net.URI; -import java.util.Properties; - -import org.junit.Test; -import static org.junit.Assert.*; - -public class TestLivyClientBuilder { - - @Test - public void testMatch() throws Exception { - Properties props = new Properties(); - props.setProperty("prop1", "_prop1_"); - props.setProperty("prop3", "prop3"); - - TestClientFactory.Client client = (TestClientFactory.Client) - new LivyClientBuilder(false) - .setURI(new URI("match")) - .setConf("prop1", "prop1") - .setConf("prop2", "prop2") - .setAll(props) - .build(); - - assertNotNull(client); - assertEquals("_prop1_", client.config.getProperty("prop1")); - assertEquals("prop2", client.config.getProperty("prop2")); - assertEquals("prop3", client.config.getProperty("prop3")); - } - - @Test(expected = IllegalArgumentException.class) - public void testMissingUri() throws Exception { - new LivyClientBuilder(false).build(); - } - - @Test(expected = IllegalArgumentException.class) - public void testMismatch() throws Exception { - assertNull(new LivyClientBuilder(false).setURI(new URI("mismatch")).build()); - } - - @Test(expected=IllegalStateException.class) - public void testFactoryError() throws Exception { - new LivyClientBuilder(false).setURI(new URI("error")).build(); - } - - @Test - public void testDefaultConfig() throws Exception { - TestClientFactory.Client client = (TestClientFactory.Client) - new LivyClientBuilder().build(); - assertEquals("override", client.config.getProperty("spark.config")); - } - - @Test - public void testRedaction() throws Exception { - try { - new LivyClientBuilder(false).setURI(new URI("mismatch://user@host")).build(); - fail("Should have failed to create client."); - } catch (IllegalArgumentException e) { - assertFalse("Should have redacted user information.", - e.getMessage().contains("user@host")); - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/test/java/org/apache/livy/TestClientFactory.java ---------------------------------------------------------------------- diff --git a/api/src/test/java/org/apache/livy/TestClientFactory.java b/api/src/test/java/org/apache/livy/TestClientFactory.java new file mode 100644 index 0000000..89edeec --- /dev/null +++ b/api/src/test/java/org/apache/livy/TestClientFactory.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.livy; + +import java.io.File; +import java.net.URI; +import java.util.Properties; +import java.util.concurrent.Future; + +public class TestClientFactory implements LivyClientFactory { + + @Override + public LivyClient createClient(URI uri, Properties config) { + switch (uri.getPath()) { + case "match": + return new Client(config); + + case "error": + throw new IllegalStateException("error"); + + default: + return null; + } + } + + public static class Client implements LivyClient { + + public final Properties config; + + private Client(Properties config) { + this.config = config; + } + + @Override + public <T> JobHandle<T> submit(Job<T> job) { + throw new UnsupportedOperationException(); + } + + @Override + public <T> Future<T> run(Job<T> job) { + throw new UnsupportedOperationException(); + } + + @Override + public void stop(boolean shutdownContext) { + throw new UnsupportedOperationException(); + } + + @Override + public Future<?> uploadJar(File jar) { + throw new UnsupportedOperationException(); + } + + @Override + public Future<?> addJar(URI uri) { + throw new UnsupportedOperationException(); + } + + @Override + public Future<?> uploadFile(File file) { + throw new UnsupportedOperationException(); + } + + @Override + public Future<?> addFile(URI uri) { + throw new UnsupportedOperationException(); + } + +} + +} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/test/java/org/apache/livy/TestLivyClientBuilder.java ---------------------------------------------------------------------- diff --git a/api/src/test/java/org/apache/livy/TestLivyClientBuilder.java b/api/src/test/java/org/apache/livy/TestLivyClientBuilder.java new file mode 100644 index 0000000..568d633 --- /dev/null +++ b/api/src/test/java/org/apache/livy/TestLivyClientBuilder.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.livy; + +import java.net.URI; +import java.util.Properties; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class TestLivyClientBuilder { + + @Test + public void testMatch() throws Exception { + Properties props = new Properties(); + props.setProperty("prop1", "_prop1_"); + props.setProperty("prop3", "prop3"); + + TestClientFactory.Client client = (TestClientFactory.Client) + new LivyClientBuilder(false) + .setURI(new URI("match")) + .setConf("prop1", "prop1") + .setConf("prop2", "prop2") + .setAll(props) + .build(); + + assertNotNull(client); + assertEquals("_prop1_", client.config.getProperty("prop1")); + assertEquals("prop2", client.config.getProperty("prop2")); + assertEquals("prop3", client.config.getProperty("prop3")); + } + + @Test(expected = IllegalArgumentException.class) + public void testMissingUri() throws Exception { + new LivyClientBuilder(false).build(); + } + + @Test(expected = IllegalArgumentException.class) + public void testMismatch() throws Exception { + assertNull(new LivyClientBuilder(false).setURI(new URI("mismatch")).build()); + } + + @Test(expected=IllegalStateException.class) + public void testFactoryError() throws Exception { + new LivyClientBuilder(false).setURI(new URI("error")).build(); + } + + @Test + public void testDefaultConfig() throws Exception { + TestClientFactory.Client client = (TestClientFactory.Client) + new LivyClientBuilder().build(); + assertEquals("override", client.config.getProperty("spark.config")); + } + + @Test + public void testRedaction() throws Exception { + try { + new LivyClientBuilder(false).setURI(new URI("mismatch://user@host")).build(); + fail("Should have failed to create client."); + } catch (IllegalArgumentException e) { + assertFalse("Should have redacted user information.", + e.getMessage().contains("user@host")); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/test/resources/META-INF/services/com.cloudera.livy.LivyClientFactory ---------------------------------------------------------------------- diff --git a/api/src/test/resources/META-INF/services/com.cloudera.livy.LivyClientFactory b/api/src/test/resources/META-INF/services/com.cloudera.livy.LivyClientFactory deleted file mode 100644 index 603e146..0000000 --- a/api/src/test/resources/META-INF/services/com.cloudera.livy.LivyClientFactory +++ /dev/null @@ -1 +0,0 @@ -com.cloudera.livy.TestClientFactory \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/api/src/test/resources/META-INF/services/org.apache.livy.LivyClientFactory ---------------------------------------------------------------------- diff --git a/api/src/test/resources/META-INF/services/org.apache.livy.LivyClientFactory b/api/src/test/resources/META-INF/services/org.apache.livy.LivyClientFactory new file mode 100644 index 0000000..99c0bd2 --- /dev/null +++ b/api/src/test/resources/META-INF/services/org.apache.livy.LivyClientFactory @@ -0,0 +1 @@ +org.apache.livy.TestClientFactory \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/assembly/pom.xml ---------------------------------------------------------------------- diff --git a/assembly/pom.xml b/assembly/pom.xml index 3b7aa6d..abe9545 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -18,14 +18,14 @@ <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> - <groupId>com.cloudera.livy</groupId> + <groupId>org.apache.livy</groupId> <artifactId>livy-main</artifactId> - <version>0.4.0-SNAPSHOT</version> + <version>0.4.0-incubating-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <artifactId>livy-assembly</artifactId> - <version>0.4.0-SNAPSHOT</version> + <version>0.4.0-incubating-SNAPSHOT</version> <packaging>pom</packaging> <properties> http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/bin/livy-server ---------------------------------------------------------------------- diff --git a/bin/livy-server b/bin/livy-server index 51d765d..ede63cd 100755 --- a/bin/livy-server +++ b/bin/livy-server @@ -92,7 +92,7 @@ start_livy_server() { LIVY_CLASSPATH="$LIVY_CLASSPATH:$YARN_CONF_DIR" fi - command="$RUNNER $LIVY_SERVER_JAVA_OPTS -cp $LIVY_CLASSPATH:$CLASSPATH com.cloudera.livy.server.LivyServer" + command="$RUNNER $LIVY_SERVER_JAVA_OPTS -cp $LIVY_CLASSPATH:$CLASSPATH org.apache.livy.server.LivyServer" if [ $1 = "old" ]; then exec $command http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/checkstyle.xml ---------------------------------------------------------------------- diff --git a/checkstyle.xml b/checkstyle.xml index 929f739..76c37a9 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -102,7 +102,7 @@ <module name="ImportOrder"> <property name="separated" value="true"/> <property name="ordered" value="true"/> - <property name="groups" value="/^javax?\./,scala,*,com.cloudera.livy"/> + <property name="groups" value="/^javax?\./,scala,*,org.apache.livy"/> </module> <module name="MethodParamPad"/> <module name="AnnotationLocation"> http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/412ccc8f/client-common/pom.xml ---------------------------------------------------------------------- diff --git a/client-common/pom.xml b/client-common/pom.xml index cdf48e6..6f40e0d 100644 --- a/client-common/pom.xml +++ b/client-common/pom.xml @@ -18,19 +18,19 @@ <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> - <groupId>com.cloudera.livy</groupId> + <groupId>org.apache.livy</groupId> <artifactId>livy-main</artifactId> - <version>0.4.0-SNAPSHOT</version> + <version>0.4.0-incubating-SNAPSHOT</version> </parent> - <groupId>com.cloudera.livy</groupId> + <groupId>org.apache.livy</groupId> <artifactId>livy-client-common</artifactId> - <version>0.4.0-SNAPSHOT</version> + <version>0.4.0-incubating-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> - <groupId>com.cloudera.livy</groupId> + <groupId>org.apache.livy</groupId> <artifactId>livy-api</artifactId> <version>${project.version}</version> </dependency>
