Repository: ambari Updated Branches: refs/heads/branch-2.6 0f4ab0a66 -> eef99836e
AMBARI-22626.Zeppelin Interpreter settings are getting updated after zeppelin restart(Prabhjyot Singh via Venkata Sairam) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/eef99836 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/eef99836 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/eef99836 Branch: refs/heads/branch-2.6 Commit: eef99836ea8f118b7735b9941890386f72d43018 Parents: 0f4ab0a Author: Venkata Sairam <venkatasairam.la...@gmail.com> Authored: Thu Dec 28 12:05:18 2017 +0530 Committer: Venkata Sairam <venkatasairam.la...@gmail.com> Committed: Thu Dec 28 12:05:18 2017 +0530 ---------------------------------------------------------------------- .../scripts/interpreter_json_template.py | 152 +++++++++++++++++++ .../package/scripts/livy2_config_template.py | 112 -------------- .../ZEPPELIN/0.7.0/package/scripts/master.py | 22 --- .../package/scripts/spark2_config_template.py | 84 ---------- 4 files changed, 152 insertions(+), 218 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/eef99836/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/interpreter_json_template.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/interpreter_json_template.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/interpreter_json_template.py index 713db23..b373e22 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/interpreter_json_template.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/interpreter_json_template.py @@ -128,6 +128,68 @@ template = ''' "isUserImpersonate": false } }, + "2C4U48MY3_spark2": { + "id": "2C4U48MY3_spark2", + "name": "spark2", + "group": "spark", + "properties": { + "spark.executor.memory": "", + "args": "", + "zeppelin.spark.printREPLOutput": "true", + "spark.cores.max": "", + "zeppelin.dep.additionalRemoteRepository": "spark-packages,http://dl.bintray.com/spark-packages/maven,false;";, + "zeppelin.spark.importImplicit": "true", + "zeppelin.spark.sql.stacktrace": "false", + "zeppelin.spark.concurrentSQL": "false", + "zeppelin.spark.useHiveContext": "true", + "zeppelin.pyspark.python": "python", + "zeppelin.dep.localrepo": "local-repo", + "zeppelin.R.knitr": "true", + "zeppelin.spark.maxResult": "1000", + "master": "local[*]", + "spark.app.name": "Zeppelin", + "zeppelin.R.image.width": "100%", + "zeppelin.R.render.options": "out.format \u003d \u0027html\u0027, comment \u003d NA, echo \u003d FALSE, results \u003d \u0027asis\u0027, message \u003d F, warning \u003d F", + "zeppelin.R.cmd": "R" + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "spark", + "class": "org.apache.zeppelin.spark.SparkInterpreter", + "defaultInterpreter": true + }, + { + "name": "sql", + "class": "org.apache.zeppelin.spark.SparkSqlInterpreter", + "defaultInterpreter": false + }, + { + "name": "dep", + "class": "org.apache.zeppelin.spark.DepInterpreter", + "defaultInterpreter": false + }, + { + "name": "pyspark", + "class": "org.apache.zeppelin.spark.PySparkInterpreter", + "defaultInterpreter": false + }, + { + "name": "r", + "class": "org.apache.zeppelin.spark.SparkRInterpreter", + "defaultInterpreter": false + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "perNoteSession": false, + "perNoteProcess": false, + "isExistingProcess": false, + "setPermission": false + } + }, "2CK8A9MEG": { "id": "2CK8A9MEG", "name": "jdbc", @@ -259,6 +321,96 @@ template = ''' "isUserImpersonate": false } }, + "2C8A4SZ9T_livy2": { + "id": "2C8A4SZ9T_livy2", + "status": "READY", + "group": "livy", + "name": "livy2", + "properties": { + "zeppelin.livy.keytab": "", + "zeppelin.livy.spark.sql.maxResult": "1000", + "livy.spark.executor.instances": "", + "livy.spark.executor.memory": "", + "livy.spark.dynamicAllocation.enabled": "", + "livy.spark.dynamicAllocation.cachedExecutorIdleTimeout": "", + "livy.spark.dynamicAllocation.initialExecutors": "", + "zeppelin.livy.session.create_timeout": "120", + "livy.spark.driver.memory": "", + "zeppelin.livy.displayAppInfo": "true", + "livy.spark.jars.packages": "", + "livy.spark.dynamicAllocation.maxExecutors": "", + "zeppelin.livy.concurrentSQL": "false", + "zeppelin.livy.principal": "", + "livy.spark.executor.cores": "", + "zeppelin.livy.url": "http://localhost:8998";, + "zeppelin.livy.pull_status.interval.millis": "1000", + "livy.spark.driver.cores": "", + "livy.spark.dynamicAllocation.minExecutors": "" + }, + "interpreterGroup": [ + { + "class": "org.apache.zeppelin.livy.LivySparkInterpreter", + "editor": { + "editOnDblClick": false, + "language": "scala" + }, + "name": "spark", + "defaultInterpreter": false + }, + { + "class": "org.apache.zeppelin.livy.LivySparkSQLInterpreter", + "editor": { + "editOnDblClick": false, + "language": "sql" + }, + "name": "sql", + "defaultInterpreter": false + }, + { + "class": "org.apache.zeppelin.livy.LivyPySparkInterpreter", + "editor": { + "editOnDblClick": false, + "language": "python" + }, + "name": "pyspark", + "defaultInterpreter": false + }, + { + "class": "org.apache.zeppelin.livy.LivyPySpark3Interpreter", + "editor": { + "editOnDblClick": false, + "language": "python" + }, + "name": "pyspark3", + "defaultInterpreter": false + }, + { + "class": "org.apache.zeppelin.livy.LivySparkRInterpreter", + "editor": { + "editOnDblClick": false, + "language": "r" + }, + "name": "sparkr", + "defaultInterpreter": false + }, + { + "name": "shared", + "class": "org.apache.zeppelin.livy.LivySharedInterpreter", + "defaultInterpreter": false + } + ], + "dependencies": [], + "option": { + "setPermission": false, + "remote": true, + "users": [], + "isExistingProcess": false, + "perUser": "scoped", + "isUserImpersonate": false, + "perNote": "shared", + "port": -1 + } + }, "2CKAY1A8Y": { "id": "2CKAY1A8Y", "name": "md", http://git-wip-us.apache.org/repos/asf/ambari/blob/eef99836/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/livy2_config_template.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/livy2_config_template.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/livy2_config_template.py deleted file mode 100644 index c9030bd..0000000 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/livy2_config_template.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -template = ''' -{ - "id": "2C8A4SZ9T_livy2", - "status": "READY", - "group": "livy", - "name": "livy2", - "properties": { - "zeppelin.livy.keytab": "", - "zeppelin.livy.spark.sql.maxResult": "1000", - "livy.spark.executor.instances": "", - "livy.spark.executor.memory": "", - "livy.spark.dynamicAllocation.enabled": "", - "livy.spark.dynamicAllocation.cachedExecutorIdleTimeout": "", - "livy.spark.dynamicAllocation.initialExecutors": "", - "zeppelin.livy.session.create_timeout": "120", - "livy.spark.driver.memory": "", - "zeppelin.livy.displayAppInfo": "true", - "livy.spark.jars.packages": "", - "livy.spark.dynamicAllocation.maxExecutors": "", - "zeppelin.livy.concurrentSQL": "false", - "zeppelin.livy.principal": "", - "livy.spark.executor.cores": "", - "zeppelin.livy.url": "http://localhost:8998";, - "zeppelin.livy.pull_status.interval.millis": "1000", - "livy.spark.driver.cores": "", - "livy.spark.dynamicAllocation.minExecutors": "" - }, - "interpreterGroup": [ - { - "class": "org.apache.zeppelin.livy.LivySparkInterpreter", - "editor": { - "editOnDblClick": false, - "language": "scala" - }, - "name": "spark", - "defaultInterpreter": false - }, - { - "class": "org.apache.zeppelin.livy.LivySparkSQLInterpreter", - "editor": { - "editOnDblClick": false, - "language": "sql" - }, - "name": "sql", - "defaultInterpreter": false - }, - { - "class": "org.apache.zeppelin.livy.LivyPySparkInterpreter", - "editor": { - "editOnDblClick": false, - "language": "python" - }, - "name": "pyspark", - "defaultInterpreter": false - }, - { - "class": "org.apache.zeppelin.livy.LivyPySpark3Interpreter", - "editor": { - "editOnDblClick": false, - "language": "python" - }, - "name": "pyspark3", - "defaultInterpreter": false - }, - { - "class": "org.apache.zeppelin.livy.LivySparkRInterpreter", - "editor": { - "editOnDblClick": false, - "language": "r" - }, - "name": "sparkr", - "defaultInterpreter": false - }, - { - "name": "shared", - "class": "org.apache.zeppelin.livy.LivySharedInterpreter", - "defaultInterpreter": false - } - ], - "dependencies": [], - "option": { - "setPermission": false, - "remote": true, - "users": [], - "isExistingProcess": false, - "perUser": "scoped", - "isUserImpersonate": false, - "perNote": "shared", - "port": -1 - } -} -''' http://git-wip-us.apache.org/repos/asf/ambari/blob/eef99836/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py index bb4c5b2..ee264a4 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py @@ -450,16 +450,6 @@ class Master(Script): config_data = self.get_interpreter_settings() interpreter_settings = config_data['interpreterSettings'] - if 'spark2-defaults' in params.config['configurations']: - spark2_config = self.get_spark2_interpreter_config() - config_id = spark2_config["id"] - interpreter_settings[config_id] = spark2_config - - if params.livy2_livyserver_host: - livy2_config = self.get_livy2_interpreter_config() - config_id = livy2_config["id"] - interpreter_settings[config_id] = livy2_config - if params.zeppelin_interpreter: settings_to_delete = [] for settings_key, interpreter in interpreter_settings.items(): @@ -612,17 +602,5 @@ class Master(Script): import params return glob.glob(params.zeppelin_dir + '/interpreter/spark/dep/zeppelin-spark-dependencies*.jar') - def get_spark2_interpreter_config(self): - import spark2_config_template - import json - - return json.loads(spark2_config_template.template) - - def get_livy2_interpreter_config(self): - import livy2_config_template - import json - - return json.loads(livy2_config_template.template) - if __name__ == "__main__": Master().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/eef99836/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/spark2_config_template.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/spark2_config_template.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/spark2_config_template.py deleted file mode 100644 index 28a63c6..0000000 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/spark2_config_template.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -template = ''' -{ - "id": "2C4U48MY3_spark2", - "name": "spark2", - "group": "spark", - "properties": { - "spark.executor.memory": "", - "args": "", - "zeppelin.spark.printREPLOutput": "true", - "spark.cores.max": "", - "zeppelin.dep.additionalRemoteRepository": "spark-packages,http://dl.bintray.com/spark-packages/maven,false;";, - "zeppelin.spark.importImplicit": "true", - "zeppelin.spark.sql.stacktrace": "false", - "zeppelin.spark.concurrentSQL": "false", - "zeppelin.spark.useHiveContext": "true", - "zeppelin.pyspark.python": "python", - "zeppelin.dep.localrepo": "local-repo", - "zeppelin.R.knitr": "true", - "zeppelin.spark.maxResult": "1000", - "master": "local[*]", - "spark.app.name": "Zeppelin", - "zeppelin.R.image.width": "100%", - "zeppelin.R.render.options": "out.format \u003d \u0027html\u0027, comment \u003d NA, echo \u003d FALSE, results \u003d \u0027asis\u0027, message \u003d F, warning \u003d F", - "zeppelin.R.cmd": "R" - }, - "status": "READY", - "interpreterGroup": [ - { - "name": "spark", - "class": "org.apache.zeppelin.spark.SparkInterpreter", - "defaultInterpreter": true - }, - { - "name": "sql", - "class": "org.apache.zeppelin.spark.SparkSqlInterpreter", - "defaultInterpreter": false - }, - { - "name": "dep", - "class": "org.apache.zeppelin.spark.DepInterpreter", - "defaultInterpreter": false - }, - { - "name": "pyspark", - "class": "org.apache.zeppelin.spark.PySparkInterpreter", - "defaultInterpreter": false - }, - { - "name": "r", - "class": "org.apache.zeppelin.spark.SparkRInterpreter", - "defaultInterpreter": false - } - ], - "dependencies": [], - "option": { - "remote": true, - "port": -1, - "perNoteSession": false, - "perNoteProcess": false, - "isExistingProcess": false, - "setPermission": false - } -} -''' \ No newline at end of file
