Repository: ambari Updated Branches: refs/heads/branch-2.5 5d511d858 -> b201accc6
AMBARI-21243.Improve Zeppelin startup time(Prabhjyot Singh via Venkata Sairam) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b201accc Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b201accc Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b201accc Branch: refs/heads/branch-2.5 Commit: b201accc602b39342a83b7c7292e248d8b60aff6 Parents: 5d511d8 Author: Venkata Sairam <[email protected]> Authored: Fri Jun 16 11:47:31 2017 +0530 Committer: Venkata Sairam <[email protected]> Committed: Fri Jun 16 11:49:19 2017 +0530 ---------------------------------------------------------------------- .../scripts/interpreter_json_template.py | 361 +++++++++++++++++++ .../0.6.0.2.5/package/scripts/master.py | 17 +- 2 files changed, 368 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b201accc/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/interpreter_json_template.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/interpreter_json_template.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/interpreter_json_template.py new file mode 100644 index 0000000..6a98919 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/interpreter_json_template.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +template = ''' +{ + "interpreterSettings": { + "2CKEKWY8Z": { + "id": "2CKEKWY8Z", + "name": "angular", + "group": "angular", + "properties": {}, + "status": "READY", + "interpreterGroup": [ + { + "name": "angular", + "class": "org.apache.zeppelin.angular.AngularInterpreter", + "defaultInterpreter": false, + "editor": { + "editOnDblClick": true + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "shared", + "isExistingProcess": false, + "setPermission": false, + "users": [], + "isUserImpersonate": false + } + }, + "2CKX8WPU1": { + "id": "2CKX8WPU1", + "name": "spark", + "group": "spark", + "properties": { + "spark.executor.memory": "512m", + "args": "", + "zeppelin.spark.printREPLOutput": "true", + "spark.cores.max": "", + "zeppelin.dep.additionalRemoteRepository": "spark-packages,http://dl.bintray.com/spark-packages/maven,false;";, + "zeppelin.spark.sql.stacktrace": "false", + "zeppelin.spark.importImplicit": "true", + "zeppelin.spark.concurrentSQL": "false", + "zeppelin.spark.useHiveContext": "true", + "zeppelin.pyspark.python": "python", + "zeppelin.dep.localrepo": "local-repo", + "zeppelin.R.knitr": "true", + "zeppelin.spark.maxResult": "1000", + "master": "yarn-client", + "spark.app.name": "Zeppelin", + "zeppelin.R.image.width": "100%", + "zeppelin.R.render.options": "out.format \u003d \u0027html\u0027, comment \u003d NA, echo \u003d FALSE, results \u003d \u0027asis\u0027, message \u003d F, warning \u003d F", + "zeppelin.R.cmd": "R" + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "spark", + "class": "org.apache.zeppelin.spark.SparkInterpreter", + "defaultInterpreter": true, + "editor": { + "language": "scala" + } + }, + { + "name": "sql", + "class": "org.apache.zeppelin.spark.SparkSqlInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sql" + } + }, + { + "name": "dep", + "class": "org.apache.zeppelin.spark.DepInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "scala" + } + }, + { + "name": "pyspark", + "class": "org.apache.zeppelin.spark.PySparkInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python" + } + }, + { + "name": "r", + "class": "org.apache.zeppelin.spark.SparkRInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "r" + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "shared", + "isExistingProcess": false, + "setPermission": false, + "users": [], + "isUserImpersonate": false + } + }, + "2CK8A9MEG": { + "id": "2CK8A9MEG", + "name": "jdbc", + "group": "jdbc", + "properties": { + "default.password": "", + "zeppelin.jdbc.auth.type": "", + "common.max_count": "1000", + "zeppelin.jdbc.principal": "", + "default.user": "gpadmin", + "default.url": "jdbc:postgresql://localhost:5432/", + "default.driver": "org.postgresql.Driver", + "zeppelin.jdbc.keytab.location": "", + "zeppelin.jdbc.concurrent.use": "true", + "zeppelin.jdbc.concurrent.max_connection": "10" + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "sql", + "class": "org.apache.zeppelin.jdbc.JDBCInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sql", + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "shared", + "isExistingProcess": false, + "setPermission": false, + "users": [], + "isUserImpersonate": false + } + }, + "2CKX6DGQZ": { + "id": "2CKX6DGQZ", + "name": "livy", + "group": "livy", + "properties": { + "zeppelin.livy.pull_status.interval.millis": "1000", + "livy.spark.executor.memory": "", + "zeppelin.livy.session.create_timeout": "120", + "zeppelin.livy.principal": "", + "zeppelin.livy.spark.sql.maxResult": "1000", + "zeppelin.livy.keytab": "", + "zeppelin.livy.concurrentSQL": "false", + "zeppelin.livy.spark.sql.field.truncate": "true", + "livy.spark.executor.cores": "", + "zeppelin.livy.displayAppInfo": "false", + "zeppelin.livy.url": "http://localhost:8998";, + "livy.spark.dynamicAllocation.minExecutors": "", + "livy.spark.driver.cores": "", + "livy.spark.jars.packages": "", + "livy.spark.dynamicAllocation.enabled": "", + "livy.spark.executor.instances": "", + "livy.spark.dynamicAllocation.cachedExecutorIdleTimeout": "", + "livy.spark.dynamicAllocation.maxExecutors": "", + "livy.spark.dynamicAllocation.initialExecutors": "", + "livy.spark.driver.memory": "" + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "spark", + "class": "org.apache.zeppelin.livy.LivySparkInterpreter", + "defaultInterpreter": true, + "editor": { + "language": "scala", + "editOnDblClick": false + } + }, + { + "name": "sql", + "class": "org.apache.zeppelin.livy.LivySparkSQLInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sql", + "editOnDblClick": false + } + }, + { + "name": "pyspark", + "class": "org.apache.zeppelin.livy.LivyPySparkInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false + } + }, + { + "name": "pyspark3", + "class": "org.apache.zeppelin.livy.LivyPySpark3Interpreter", + "defaultInterpreter": false, + "editor": { + "language": "python", + "editOnDblClick": false + } + }, + { + "name": "sparkr", + "class": "org.apache.zeppelin.livy.LivySparkRInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "r", + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "scoped", + "isExistingProcess": false, + "setPermission": false, + "users": [], + "isUserImpersonate": false + } + }, + "2CKAY1A8Y": { + "id": "2CKAY1A8Y", + "name": "md", + "group": "md", + "properties": { + "markdown.parser.type": "pegdown" + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "md", + "class": "org.apache.zeppelin.markdown.Markdown", + "defaultInterpreter": false, + "editor": { + "language": "markdown", + "editOnDblClick": true + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "shared", + "isExistingProcess": false, + "setPermission": false, + "users": [], + "isUserImpersonate": false + } + }, + "2CHS8UYQQ": { + "id": "2CHS8UYQQ", + "name": "sh", + "group": "sh", + "properties": { + "zeppelin.shell.keytab.location": "", + "shell.command.timeout.millisecs": "60000", + "zeppelin.shell.principal": "", + "zeppelin.shell.auth.type": "" + }, + "status": "READY", + "interpreterGroup": [ + { + "name": "sh", + "class": "org.apache.zeppelin.shell.ShellInterpreter", + "defaultInterpreter": false, + "editor": { + "language": "sh", + "editOnDblClick": false + } + } + ], + "dependencies": [], + "option": { + "remote": true, + "port": -1, + "perNote": "shared", + "perUser": "shared", + "isExistingProcess": false, + "setPermission": false, + "users": [], + "isUserImpersonate": false + } + } + }, + "interpreterBindings": {}, + "interpreterRepositories": [ + { + "id": "central", + "type": "default", + "url": "http://repo1.maven.org/maven2/";, + "releasePolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "snapshotPolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "mirroredRepositories": [], + "repositoryManager": false + }, + { + "id": "local", + "type": "default", + "url": "file:///home/zeppelin/.m2/repository", + "releasePolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "snapshotPolicy": { + "enabled": true, + "updatePolicy": "daily", + "checksumPolicy": "warn" + }, + "mirroredRepositories": [], + "repositoryManager": false + } + ] +} +''' http://git-wip-us.apache.org/repos/asf/ambari/blob/b201accc/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py index 3d516b0..75ca6a5 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py @@ -217,9 +217,7 @@ class Master(Script): # if first_setup: if not glob.glob(params.conf_dir + "/interpreter.json") and \ not os.path.exists(params.conf_dir + "/interpreter.json"): - Execute(params.zeppelin_dir + '/bin/zeppelin-daemon.sh start >> ' - + params.zeppelin_log_file, user=params.zeppelin_user) - self.check_zeppelin_server() + self.create_interpreter_json() self.update_zeppelin_interpreter() self.update_kerberos_properties() @@ -421,14 +419,13 @@ class Master(Script): self.set_interpreter_settings(config_data) - @retry(times=30, sleep_time=5, err_class=Fail) - def check_zeppelin_server(self): + def create_interpreter_json(self): + import interpreter_json_template import params - path = params.conf_dir + "/interpreter.json" - if os.path.exists(path) and os.path.getsize(path): - Logger.info("interpreter.json found. Zeppelin server started.") - else: - raise Fail("interpreter.json not found. waiting for Zeppelin server to start...") + + interpreter_json = interpreter_json_template.template + File(format("{params.conf_dir}/interpreter.json"), content=interpreter_json, + owner=params.zeppelin_user, group=params.zeppelin_group) def get_zeppelin_spark_dependencies(self): import params
