Repository: ambari Updated Branches: refs/heads/branch-2.6 f49cf54e9 -> c7b9348da
AMBARI-21987.Store Zeppelin's interpreter.json in HDFS(Prabhjyot Singh via Venkata Sairam) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c7b9348d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c7b9348d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c7b9348d Branch: refs/heads/branch-2.6 Commit: c7b9348da34ce7d78c37eb93b254cdaf396bc061 Parents: f49cf54 Author: Venkata Sairam <venkatasairam.la...@gmail.com> Authored: Wed Sep 20 12:02:29 2017 +0530 Committer: Venkata Sairam <venkatasairam.la...@gmail.com> Committed: Wed Sep 20 12:02:29 2017 +0530 ---------------------------------------------------------------------- .../0.6.0/configuration/zeppelin-config.xml | 2 +- .../ZEPPELIN/0.6.0/package/scripts/master.py | 2 +- .../0.7.0/configuration/zeppelin-config.xml | 8 ++- .../ZEPPELIN/0.7.0/package/scripts/master.py | 67 ++++++++++++++++---- 4 files changed, 63 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c7b9348d/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml index bd6ad76..03ad5f7 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml @@ -77,7 +77,7 @@ </property> <property> <name>zeppelin.notebook.storage</name> - <value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value> + <value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value> <description>notebook persistence layer implementation. If S3 is used, set this to org.apache.zeppelin.notebook.repo.S3NotebookRepo instead. If S3 is used to store the notebooks, it is necessary to use the following folder structure http://git-wip-us.apache.org/repos/asf/ambari/blob/c7b9348d/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/master.py index ba73d10..0fdc27c 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/master.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/master.py @@ -242,7 +242,7 @@ class Master(Script): os.path.join(params.zeppelin_dir, "notebook")), sudo=True) if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config'] \ - and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.HdfsNotebookRepo': + and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo': self.check_and_copy_notebook_in_hdfs(params) if params.security_enabled: http://git-wip-us.apache.org/repos/asf/ambari/blob/c7b9348d/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml index bd6ad76..29821f2 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml @@ -77,7 +77,7 @@ </property> <property> <name>zeppelin.notebook.storage</name> - <value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value> + <value>org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo</value> <description>notebook persistence layer implementation. If S3 is used, set this to org.apache.zeppelin.notebook.repo.S3NotebookRepo instead. If S3 is used to store the notebooks, it is necessary to use the following folder structure @@ -86,6 +86,12 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>zeppelin.config.fs.dir</name> + <value>conf</value> + <description>Location where interpreter.json should be installed</description> + <on-ambari-upgrade add="false"/> + </property> + <property> <name>zeppelin.interpreter.dir</name> <value>interpreter</value> <description>Interpreter implementation base directory</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/c7b9348d/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py index ba73d10..7d44d16 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py @@ -23,6 +23,7 @@ import os from resource_management.core import shell, sudo from resource_management.core.logger import Logger +from resource_management.core.exceptions import Fail from resource_management.core.resources import Directory from resource_management.core.resources.system import Execute, File from resource_management.core.source import InlineTemplate @@ -242,7 +243,7 @@ class Master(Script): os.path.join(params.zeppelin_dir, "notebook")), sudo=True) if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config'] \ - and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.HdfsNotebookRepo': + and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo': self.check_and_copy_notebook_in_hdfs(params) if params.security_enabled: @@ -294,23 +295,49 @@ class Master(Script): self.set_interpreter_settings(config_data) + def pre_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing Stack Upgrade pre-restart") + import params + env.set_params(params) + + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)): + stack_select.select_packages(params.version) + + def getZeppelinConfFS(self, params): + hdfs_interpreter_config = params.config['configurations']['zeppelin-config']['zeppelin.config.fs.dir'] + "/interpreter.json" + + if not hdfs_interpreter_config.startswith("/"): + hdfs_interpreter_config = "/user/" + format("{zeppelin_user}") + "/" + hdfs_interpreter_config + + return hdfs_interpreter_config + def get_interpreter_settings(self): import params import json interpreter_config = os.path.join(params.conf_dir, "interpreter.json") + if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config'] \ + and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo': + + if 'zeppelin.config.fs.dir' in params.config['configurations']['zeppelin-config']: + try: + # copy from hdfs to /etc/zeppelin/conf/interpreter.json + params.HdfsResource(interpreter_config, + type="file", + action="download_on_execute", + source=self.getZeppelinConfFS(params), + group=params.zeppelin_group, + user=params.zeppelin_user) + except Fail as fail: + if "doesn't exist" not in fail.args[0]: + print "Error getting interpreter.json from HDFS" + print fail.args + raise Fail + config_content = sudo.read_file(interpreter_config) config_data = json.loads(config_content) return config_data - def pre_upgrade_restart(self, env, upgrade_type=None): - Logger.info("Executing Stack Upgrade pre-restart") - import params - env.set_params(params) - - if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)): - stack_select.select_packages(params.version) - def set_interpreter_settings(self, config_data): import params import json @@ -319,8 +346,19 @@ class Master(Script): File(interpreter_config, group=params.zeppelin_group, owner=params.zeppelin_user, - content=json.dumps(config_data, indent=2) - ) + content=json.dumps(config_data, indent=2)) + + if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config'] \ + and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo': + + if 'zeppelin.config.fs.dir' in params.config['configurations']['zeppelin-config']: + params.HdfsResource(self.getZeppelinConfFS(params), + type="file", + action="create_on_execute", + source=interpreter_config, + group=params.zeppelin_group, + user=params.zeppelin_user, + replace_existing_files=True) def update_kerberos_properties(self): import params @@ -499,8 +537,11 @@ class Master(Script): import params interpreter_json = interpreter_json_template.template - File(format("{params.conf_dir}/interpreter.json"), content=interpreter_json, - owner=params.zeppelin_user, group=params.zeppelin_group) + File(format("{params.conf_dir}/interpreter.json"), + content=interpreter_json, + owner=params.zeppelin_user, + group=params.zeppelin_group, + mode=0664) def get_zeppelin_spark_dependencies(self): import params