Repository: incubator-toree Updated Branches: refs/heads/master 846292233 -> 05faf9f5b
[TOREE 249] Enable pip install This commit enables the building, packaging and release of a pip package. The package can be installed via `pip install toree`. This then enables the kernel installation via `jupyter toree install`. Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/05faf9f5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/05faf9f5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/05faf9f5 Branch: refs/heads/master Commit: 05faf9f5bd5083b21fa9ede80a09092e0a841829 Parents: 8462922 Author: Corey A. Stubbs <cstu...@us.ibm.com> Authored: Wed Feb 3 10:36:42 2016 -0600 Committer: Corey A. Stubbs <cstu...@us.ibm.com> Committed: Thu Feb 4 10:52:49 2016 -0600 ---------------------------------------------------------------------- .gitignore | 5 +- Makefile | 72 +++++++++++++---- README.md | 22 ++++-- Vagrantfile | 6 +- etc/bin/run.sh | 36 +++++++++ etc/bin/toree-kernel | 36 --------- etc/pip_install/MANIFEST.in | 3 + etc/pip_install/setup.py | 69 +++++++++++++++++ etc/pip_install/toree/__init__.py | 18 +++++ etc/pip_install/toree/__main__.py | 21 +++++ etc/pip_install/toree/_version.py | 18 +++++ etc/pip_install/toree/toreeapp.py | 138 +++++++++++++++++++++++++++++++++ 12 files changed, 383 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/05faf9f5/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 12db42c..c5eead5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ - +build/ +**/derby.log +**/metastore_db +**/*__pycache__ **.swp target/ .idea/ http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/05faf9f5/Makefile ---------------------------------------------------------------------- diff --git a/Makefile b/Makefile index 1e90a4c..fbba6f7 100644 --- a/Makefile +++ b/Makefile @@ -15,16 +15,33 @@ # limitations under the License # -.PHONY: help clean clean-dist build dev test test-travis +.PHONY: help clean clean-dist build dev test test-travis release pip-release bin-release -VERSION?=0.1.0 +VERSION?=0.1.0.dev2 +COMMIT=$(shell git rev-parse --short=12 --verify HEAD) +ifeq (, $(findstring dev, $(VERSION))) +IS_SNAPSHOT?=false +else IS_SNAPSHOT?=true -APACHE_SPARK_VERSION?=1.5.1 - -ifeq ($(IS_SNAPSHOT),true) SNAPSHOT:=-SNAPSHOT endif +APACHE_SPARK_VERSION?=1.5.1 +IMAGE?=jupyter/pyspark-notebook:2988869079e6 +DOCKER_WORKDIR?=/srv/toree +DOCKER_ARGS?= +define DOCKER +docker run -it --rm \ + --workdir $(DOCKER_WORKDIR) \ + -e PYTHONPATH='/srv/toree' \ + -v `pwd`:/srv/toree $(DOCKER_ARGS) +endef + +define GEN_PIP_PACKAGE_INFO +printf "__version__ = '$(VERSION)'\n" >> dist/toree/_version.py +printf "__commit__ = '$(COMMIT)'\n" >> dist/toree/_version.py +endef + USE_VAGRANT?= RUN_PREFIX=$(if $(USE_VAGRANT),vagrant ssh -c "cd $(VM_WORKDIR) && ) RUN_SUFFIX=$(if $(USE_VAGRANT),") @@ -33,18 +50,19 @@ RUN=$(RUN_PREFIX)$(1)$(RUN_SUFFIX) ENV_OPTS:=APACHE_SPARK_VERSION=$(APACHE_SPARK_VERSION) VERSION=$(VERSION) IS_SNAPSHOT=$(IS_SNAPSHOT) -FULL_VERSION:=$(VERSION)$(SNAPSHOT) -ASSEMBLY_JAR:=toree-kernel-assembly-$(FULL_VERSION).jar +ASSEMBLY_JAR:=toree-kernel-assembly-$(VERSION)$(SNAPSHOT).jar help: @echo ' clean - clean build files' @echo ' dev - starts ipython' - @echo ' dist - build a packaged distribution' + @echo ' dist - build a directory with contents to package' @echo ' build - builds assembly' @echo ' test - run all units' + @echo ' release - creates packaged distribution' + @echo ' jupyter - starts a Jupyter Notebook with Toree installed' build-info: - @echo '$(ENV_OPTS) $(FULL_VERSION)' + @echo '$(ENV_OPTS) $(VERSION)' clean-dist: -rm -r dist @@ -69,17 +87,39 @@ test: VM_WORKDIR=/src/toree-kernel test: $(call RUN,$(ENV_OPTS) sbt compile test) -dist: COMMIT=$(shell git rev-parse --short=12 --verify HEAD) -dist: VERSION_FILE=dist/toree-kernel/VERSION +dist: VERSION_FILE=dist/toree/VERSION dist: kernel/target/scala-2.10/$(ASSEMBLY_JAR) ${shell find ./etc/bin/*} - @mkdir -p dist/toree-kernel/bin dist/toree-kernel/lib - @cp -r etc/bin/* dist/toree-kernel/bin/. - @cp kernel/target/scala-2.10/$(ASSEMBLY_JAR) dist/toree-kernel/lib/. - @echo "VERSION: $(FULL_VERSION)" > $(VERSION_FILE) + @mkdir -p dist/toree/bin dist/toree/lib + @cp -r etc/bin/* dist/toree/bin/. + @cp kernel/target/scala-2.10/$(ASSEMBLY_JAR) dist/toree/lib/. + @echo "VERSION: $(VERSION)" > $(VERSION_FILE) @echo "COMMIT: $(COMMIT)" >> $(VERSION_FILE) - @cd dist; tar -cvzf toree-kernel-$(FULL_VERSION).tar.gz toree-kernel test-travis: $(ENV_OPTS) sbt clean test -Dakka.test.timefactor=3 find $(HOME)/.sbt -name "*.lock" | xargs rm find $(HOME)/.ivy2 -name "ivydata-*.properties" | xargs rm + +pip-release: DOCKER_WORKDIR=/srv/toree/dist +pip-release: dist + @cp -rf etc/pip_install/* dist/. + @$(GEN_PIP_PACKAGE_INFO) + @$(DOCKER) $(IMAGE) python setup.py sdist --dist-dir=. + @$(DOCKER) -p 8888:8888 --user=root $(IMAGE) bash -c 'pip install toree-$(VERSION).tar.gz && jupyter toree install' + +bin-release: dist + @(cd dist; tar -cvzf toree-$(VERSION)-binary-release.tar.gz toree) + +release: DOCKER_WORKDIR=/srv/toree/dist +release: PYPI_REPO?=https://pypi.python.org/pypi +release: PYPI_USER?= +release: PYPI_PASSWORD?= +release: PYPIRC=printf "[distutils]\nindex-servers =\n\tpypi\n\n[pypi]\nrepository: $(PYPI_REPO) \nusername: $(PYPI_USER)\npassword: $(PYPI_PASSWORD)" > ~/.pypirc; +release: pip-release bin-release + @$(DOCKER) $(IMAGE) bash -c '$(PYPIRC) pip install twine && \ + python setup.py register -r $(PYPI_REPO) && \ + twine upload -r pypi toree-$(VERSION).tar.gz' + +jupyter: DOCKER_WORKDIR=/srv/toree/dist +jupyter: pip-release + @$(DOCKER) -p 8888:8888 --user=root $(IMAGE) bash -c 'pip install toree-$(VERSION).tar.gz && jupyter toree install && cd ~ && jupyter notebook --ip=* --no-browser' http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/05faf9f5/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index fe0e944..0d51279 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ A version of Toree is deployed as part of the [Try Jupyter!][try-jupyter] site. Develop ======= This project uses `make` as the entry point for build, test, and packaging. It supports 2 modes, local and vagrant. The default is local and all command (i.e. sbt) will be ran locally on your machine. This means that you need to -install `sbt`, `jupyter/ipython`, and other develoment requirements locally on your machine. The 2nd mode uses [Vagrant][vagrant] to simplify the development experience. In vagrant mode, all commands are sent to the vagrant box +install `sbt`, `jupyter/ipython`, and other development requirements locally on your machine. The 2nd mode uses [Vagrant][vagrant] to simplify the development experience. In vagrant mode, all commands are sent to the vagrant box that has all necessary dependencies pre-installed. To run in vagrant mode, run `export USE_VAGRANT=true`. To build and interact with Toree using Jupyter, run @@ -42,10 +42,22 @@ Build & Package =============== To build and package up Toree, run ``` -make dist +make release ``` -The resulting package of the kernel will be located at `./dist//toree-kernel-<VERSION>.tar.gz`. The uncompressed package is what is used is ran by Jupyter when doing `make dev`. +This results in 2 packages. + +- `./dist/toree-<VERSION>-binary-release.tar.gz` is a simple package that contains JAR and executable +- `./dist/toree-<VERSION>.tar.gz` is a `pip` installable package that adds Toree as a Jupyter kernel. + +NOTE: `make release` uses `docker`. Please refer to `docker` installation instructions for your system. `USE_VAGRANT` is not supported by this `make` target. + +Install +======= +``` +pip install toree +jupyter toree install +``` Reporting Issues ================ @@ -57,9 +69,9 @@ You can reach us through [gitter][gitter-url] or our [mailing list][mail-list] Version ======= -We are working on publishing binary releases of Toree soon. As part of our move into Apache Incubator, Toree will start a new version sequence starting at `0.1`. +We are working on publishing binary releases of Toree soon. As part of our move into Apache Incubator, Toree will start a new version sequence starting at `0.1`. -Our goal is to keep `master` up to date with the latest version of Spark. When new versions of Spark require specific code changes to Toree, we will branch out older Spark version support. +Our goal is to keep `master` up to date with the latest version of Spark. When new versions of Spark require specific code changes to Toree, we will branch out older Spark version support. As it stands, we maintain several branches for legacy versions of Spark. The table below shows what is available now. http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/05faf9f5/Vagrantfile ---------------------------------------------------------------------- diff --git a/Vagrantfile b/Vagrantfile index dddce07..499194f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -121,10 +121,10 @@ echo "Adding kernel.json" mkdir -p /home/vagrant/.ipython/kernels/toree-kernel cat << EOF > /home/vagrant/.ipython/kernels/toree-kernel/kernel.json { - "display_name": "Spark 1.5.1 (Scala 2.10.4)", + "display_name": "Toree", "language_info": { "name": "scala" }, "argv": [ - "/src/toree-kernel/dist/toree-kernel/bin/toree-kernel", + "/src/toree-kernel/dist/toree/bin/run.sh", "--profile", "{connection_file}" ], @@ -157,7 +157,7 @@ CodeMirror.requireMode('clike',function(){ scalaConf[prop] = parserConf[prop]; } } - + scalaConf.name = 'text/x-scala'; var mode = CodeMirror.getMode(conf, scalaConf); http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/05faf9f5/etc/bin/run.sh ---------------------------------------------------------------------- diff --git a/etc/bin/run.sh b/etc/bin/run.sh new file mode 100755 index 0000000..bdeb402 --- /dev/null +++ b/etc/bin/run.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +# + `` +PROG_HOME="$(cd "`dirname "$0"`"/..; pwd)" + +if [ -z "$SPARK_HOME" ]; then + echo "SPARK_HOME must be set to the location of a Spark distribution!" + exit 1 +fi + +echo "Starting Spark Kernel with SPARK_HOME=$SPARK_HOME" + +KERNEL_ASSEMBLY=`(cd ${PROG_HOME}/lib; ls -1 toree-kernel-assembly-*.jar;)` + +# disable randomized hash for string in Python 3.3+ +export PYTHONHASHSEED=0 + +exec "$SPARK_HOME"/bin/spark-submit \ + ${SPARK_OPTS} \ + --class org.apache.toree.Main $PROG_HOME/lib/${KERNEL_ASSEMBLY} "$@" http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/05faf9f5/etc/bin/toree-kernel ---------------------------------------------------------------------- diff --git a/etc/bin/toree-kernel b/etc/bin/toree-kernel deleted file mode 100755 index bdeb402..0000000 --- a/etc/bin/toree-kernel +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License -# - `` -PROG_HOME="$(cd "`dirname "$0"`"/..; pwd)" - -if [ -z "$SPARK_HOME" ]; then - echo "SPARK_HOME must be set to the location of a Spark distribution!" - exit 1 -fi - -echo "Starting Spark Kernel with SPARK_HOME=$SPARK_HOME" - -KERNEL_ASSEMBLY=`(cd ${PROG_HOME}/lib; ls -1 toree-kernel-assembly-*.jar;)` - -# disable randomized hash for string in Python 3.3+ -export PYTHONHASHSEED=0 - -exec "$SPARK_HOME"/bin/spark-submit \ - ${SPARK_OPTS} \ - --class org.apache.toree.Main $PROG_HOME/lib/${KERNEL_ASSEMBLY} "$@" http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/05faf9f5/etc/pip_install/MANIFEST.in ---------------------------------------------------------------------- diff --git a/etc/pip_install/MANIFEST.in b/etc/pip_install/MANIFEST.in new file mode 100644 index 0000000..c57ec25 --- /dev/null +++ b/etc/pip_install/MANIFEST.in @@ -0,0 +1,3 @@ +recursive-include toree/bin * +recursive-include toree/lib * +include toree/VERSION http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/05faf9f5/etc/pip_install/setup.py ---------------------------------------------------------------------- diff --git a/etc/pip_install/setup.py b/etc/pip_install/setup.py new file mode 100644 index 0000000..25388a6 --- /dev/null +++ b/etc/pip_install/setup.py @@ -0,0 +1,69 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import sys +from setuptools import setup + +here = os.path.abspath(os.path.dirname(__file__)) + +version_ns = {} +with open(os.path.join(here, 'toree', '_version.py')) as f: + exec(f.read(), {}, version_ns) + +setup_args = dict( + name='toree', + author='Toree Development Team', + author_email='u...@toree.incubator.apache.org', + description='A Jupyter kernel for enabling remote applications to interaction with Apache Spark.', + long_description = ''' + A python package for installing the Toree kernel. This will install a Jupyter + application which can be invoked to install the kernel. + ''', + url='http://toree.incubator.apache.org/', + version=version_ns['__version__'], + license='Apache License 2.0', + platforms=[], + packages=['toree'], + include_package_data=True, + install_requires=[ + 'jupyter_core>=4.0, <5.0', + 'jupyter_client>=4.0, <5.0', + 'traitlets>=4.0, <5.0' + ], + data_files=[], + classifiers=[ + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python' + ] +) + +if 'setuptools' in sys.modules: + # setupstools turns entrypoint scripts into executables on windows + setup_args['entry_points'] = { + 'console_scripts': [ + 'jupyter-toree = toree.toreeapp:main' + ] + } + # Don't bother installing the .py scripts if if we're using entrypoints + setup_args.pop('scripts', None) + +if __name__ == '__main__': + setup(**setup_args) http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/05faf9f5/etc/pip_install/toree/__init__.py ---------------------------------------------------------------------- diff --git a/etc/pip_install/toree/__init__.py b/etc/pip_install/toree/__init__.py new file mode 100644 index 0000000..13451f0 --- /dev/null +++ b/etc/pip_install/toree/__init__.py @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .toreeapp import main http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/05faf9f5/etc/pip_install/toree/__main__.py ---------------------------------------------------------------------- diff --git a/etc/pip_install/toree/__main__.py b/etc/pip_install/toree/__main__.py new file mode 100644 index 0000000..badfe71 --- /dev/null +++ b/etc/pip_install/toree/__main__.py @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import absolute_import +if __name__ == '__main__': + from toree.toreeapp import main + main() http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/05faf9f5/etc/pip_install/toree/_version.py ---------------------------------------------------------------------- diff --git a/etc/pip_install/toree/_version.py b/etc/pip_install/toree/_version.py new file mode 100644 index 0000000..d2565b1 --- /dev/null +++ b/etc/pip_install/toree/_version.py @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This file is populated when doing a make release. It should be empty by defaut. http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/05faf9f5/etc/pip_install/toree/toreeapp.py ---------------------------------------------------------------------- diff --git a/etc/pip_install/toree/toreeapp.py b/etc/pip_install/toree/toreeapp.py new file mode 100644 index 0000000..a198b01 --- /dev/null +++ b/etc/pip_install/toree/toreeapp.py @@ -0,0 +1,138 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the 'License'); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os.path +import sys +import json +from traitlets import Unicode, Dict +from jupyter_client.kernelspecapp import InstallKernelSpec +from jupyter_core.application import base_aliases +from traitlets.config.application import Application +from toree._version import __version__, __commit__ + +KERNEL_SPEC= { + 'display_name': '', + 'language_info': { 'name': 'scala' }, + 'argv': [ + '', + '--profile', + '{connection_file}' + ], + 'codemirror_mode': 'scala', + 'env': { + 'SPARK_OPTS': '', + 'MAX_INTERPRETER_THREADS': '16', + 'CAPTURE_STANDARD_OUT': 'true', + 'CAPTURE_STANDARD_ERR': 'true', + 'SEND_EMPTY_OUTPUT': 'false', + 'SPARK_HOME': '', + 'PYTHONPATH': '{0}/python:{0}/python/lib/py4j-0.8.2.1-src.zip' + } +} + +PYTHON_PATH = 'PYTHONPATH' +SPARK_HOME ='SPARK_HOME' +SPARK_OPTS = 'SPARK_OPTS' +DISPLAY_NAME='display_name' +ARGV='argv' +ENV = 'env' + +class ToreeInstall(InstallKernelSpec): + '''CLI for extension management.''' + name = u'jupyter kernel toree' + description = u'A Jupyter kernel for talking to spark' + examples = ''' + jupyter toree install + jupyter toree install --spark_home=/spark/home/dir + jupyter toree install --spark_opts='--master=local[4]' + jupyter toree install --kernel_name=toree_special + ''' + + spark_home = Unicode('/usr/local/spark', config=True, + help='''Specify where the spark files can be found.''' + ) + kernel_name = Unicode('Toree', config=True, + help='Install the kernel spec with this name and is used as the display name in jupyter' + ) + aliases = { + 'kernel_name': 'ToreeInstall.kernel_name', + 'spark_home': 'ToreeInstall.spark_home', + 'spark_opts': 'ToreeInstall.spark_opts' + } + spark_opts = Unicode('--master=local[2] --driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info', config=True, + help='''Specify command line arguments to proxy for spark config.''' + ) + + aliases.update(base_aliases) + def parse_command_line(self, argv): + super(InstallKernelSpec, self).parse_command_line(argv) + + def create_kernel_json(self, location): + KERNEL_SPEC[DISPLAY_NAME] = self.kernel_name + KERNEL_SPEC[ENV][SPARK_OPTS] = self.spark_opts + KERNEL_SPEC[ENV][SPARK_HOME] = self.spark_home + KERNEL_SPEC[ARGV][0] = os.path.join(location, 'bin', 'run.sh') + KERNEL_SPEC[ENV][PYTHON_PATH] = KERNEL_SPEC[ENV][PYTHON_PATH].format(self.spark_home) + kernel_json_file = os.path.join(location, 'kernel.json') + with open(kernel_json_file, 'w+') as f: + json.dump(KERNEL_SPEC, f, indent=2) + + def start(self): + self.log.info('Installing toree kernel') + here = os.path.abspath(os.path.join(os.path.dirname(__file__))) + parent_dir = os.path.abspath(os.path.join(here, os.pardir)) + self.sourcedir = here + install_dir = self.kernel_spec_manager.install_kernel_spec(self.sourcedir, + kernel_name=self.kernel_name, + user=self.user, + prefix=self.prefix, + replace=self.replace, + ) + self.create_kernel_json(install_dir) + + +class ToreeApp(Application): + version = __version__ + name = 'jupyter toree' + description = '''Functions for managing the Toree kernel. + This package was built with the following versions of Toree and Spark: + + \tToree Version: {} + \tToree Build Commit: {} + '''.format(__version__, __commit__) + examples = ''' + jupyter toree install - Installs the kernel as a Jupyter Kernel. + ''' + subcommands = Dict({ + 'install': (ToreeInstall, ToreeInstall.description.splitlines()[0]), + }) + + aliases = {} + flags = {} + + def start(self): + if self.subapp is None: + print('No subcommand specified. Must specify one of: %s'% list(self.subcommands)) + print() + self.print_description() + self.print_subcommands() + self.exit(1) + else: + return self.subapp.start() + +def main(): + ToreeApp.launch_instance()
