[TOREE-480] Add support for spark-context-initialization-mode none Deprecate nosparkcontext in favor of spark-context-initialization-mode none. Note that old usage of nosparkcontext is automatically converted to use spark-context-initialization-mode none.
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/b8cca2cb Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/b8cca2cb Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/b8cca2cb Branch: refs/heads/master Commit: b8cca2cb55fc17359a6cb89477bb58bef0962196 Parents: bff1c6b Author: Luciano Resende <lrese...@apache.org> Authored: Thu Aug 2 13:45:31 2018 -0700 Committer: Luciano Resende <lrese...@apache.org> Committed: Wed Aug 15 20:24:07 2018 -0700 ---------------------------------------------------------------------- etc/pip_install/toree/toreeapp.py | 2 +- .../org/apache/toree/boot/CommandLineOptions.scala | 14 ++++++++------ .../apache/toree/boot/CommandLineOptionsSpec.scala | 15 ++++++++++++++- 3 files changed, 23 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b8cca2cb/etc/pip_install/toree/toreeapp.py ---------------------------------------------------------------------- diff --git a/etc/pip_install/toree/toreeapp.py b/etc/pip_install/toree/toreeapp.py index 84dad4c..c4dc11d 100644 --- a/etc/pip_install/toree/toreeapp.py +++ b/etc/pip_install/toree/toreeapp.py @@ -49,7 +49,7 @@ class ToreeInstall(InstallKernelSpec): jupyter toree install --spark_home=/spark/home/dir jupyter toree install --spark_opts='--master=local[4]' jupyter toree install --kernel_name=toree_special - jupyter toree install --toree_opts='--nosparkcontext' + jupyter toree install --toree_opts='--spark-context-initialization-mode none' jupyter toree install --interpreters=PySpark,SQL jupyter toree install --python=python ''' http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b8cca2cb/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala ---------------------------------------------------------------------- diff --git a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala index ae7f220..39376ae 100644 --- a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala +++ b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala @@ -93,14 +93,15 @@ class CommandLineOptions(args: Seq[String]) { .withRequiredArg().ofType(classOf[String]) private val _nosparkcontext = - parser.accepts("nosparkcontext", "kernel should not create a spark context") + parser.accepts("nosparkcontext", "Deprecated (see spark-context-initialization-mode). \n Kernel should not create a spark context") private val _spark_context_initialization_mode = parser.accepts( "spark-context-initialization-mode", "Identify how the Spark context initialization occurs. " + + "NONE will not initialize a Spark Context," + "EAGER initialization will happen during runtime initialization, " + - "LAZY initialization will happen when the context is used for the first time ." - ).withRequiredArg().ofType(classOf[String]).withValuesConvertedBy( regex("(lazy)|(eager)")).defaultsTo("lazy") + "LAZY initialization will happen when the context is used for the first time." + ).withRequiredArg().ofType(classOf[String]).withValuesConvertedBy( regex("(none)|(lazy)|(eager)")).defaultsTo("lazy") private val _spark_context_initialization_timeout = parser.accepts( "spark-context-initialization-timeout", @@ -171,9 +172,10 @@ class CommandLineOptions(args: Seq[String]) { "alternate_sigint" -> get(_alternate_sigint), "jar_dir" -> get(_jar_dir), "default_interpreter" -> get(_default_interpreter), - "nosparkcontext" -> (if (has(_nosparkcontext)) Some(true) else Some(false)), - "spark_context_initialization_mode" -> (if( has(_spark_context_initialization_mode)) - get(_spark_context_initialization_mode) else Some("lazy")), + // deprecated in favor of spark-context-initialization-mode none + // "nosparkcontext" -> (if (has(_nosparkcontext)) Some(true) else Some(false)), + "spark_context_initialization_mode" -> (if( has(_spark_context_initialization_mode)) get(_spark_context_initialization_mode) + else ( if (has(_nosparkcontext)) Some("none") else Some("lazy"))), "spark_context_initialization_timeout" -> get(_spark_context_initialization_timeout), "interpreter_plugins" -> interpreterPlugins, "default_repositories" -> getAll(_default_repositories).map(_.asJava) http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b8cca2cb/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala ---------------------------------------------------------------------- diff --git a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala index 029f46b..2ec7020 100644 --- a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala +++ b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala @@ -387,7 +387,20 @@ class CommandLineOptionsSpec extends FunSpec with Matchers { config.getString(key) should be("eager") } - it("when an invalid value is specified, an exception must be thrown") { + it("when the option nosparkcontext is specified, it should properly set spark-context-initialization-mode to none") { + val options = new CommandLineOptions(List( + "--stdin-port", "99999", + "--shell-port", "88888", + "--iopub-port", "77777", + "--control-port", "55555", + "--heartbeat-port", "44444", + "--nosparkcontext", "foo" + )) + val config: Config = options.toConfig + config.getString(key) should be("none") + } + + it("when an invalid value is specified, an exception must be thrown") { intercept [OptionException] { val options = new CommandLineOptions(List( "--stdin-port", "99999",
