AMBARI-21835.Support for Zeppelin notebook storage in HDFS(Prabhjyot Singh via Venkata Sairam)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a679281f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a679281f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a679281f Branch: refs/heads/feature-branch-AMBARI-21307 Commit: a679281f7abe6768e0896e32df7c298b59877064 Parents: 7e22217 Author: Venkata Sairam <venkatasairam.la...@gmail.com> Authored: Mon Sep 4 14:44:25 2017 +0530 Committer: Venkata Sairam <venkatasairam.la...@gmail.com> Committed: Mon Sep 4 14:45:22 2017 +0530 ---------------------------------------------------------------------- .../0.6.0.2.5/configuration/zeppelin-config.xml | 2 +- .../0.6.0.2.5/package/scripts/master.py | 59 +++++++++++++++++--- .../0.6.0.2.5/package/scripts/params.py | 2 + .../0.6.0.3.0/configuration/zeppelin-config.xml | 2 +- 4 files changed, 55 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a679281f/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml index 03ad5f7..bd6ad76 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml @@ -77,7 +77,7 @@ </property> <property> <name>zeppelin.notebook.storage</name> - <value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value> + <value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value> <description>notebook persistence layer implementation. If S3 is used, set this to org.apache.zeppelin.notebook.repo.S3NotebookRepo instead. If S3 is used to store the notebooks, it is necessary to use the following folder structure http://git-wip-us.apache.org/repos/asf/ambari/blob/a679281f/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py index 2142bb4..ba73d10 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py @@ -20,25 +20,25 @@ limitations under the License. import glob import os -from resource_management.core.base import Fail + +from resource_management.core import shell, sudo +from resource_management.core.logger import Logger from resource_management.core.resources import Directory from resource_management.core.resources.system import Execute, File from resource_management.core.source import InlineTemplate -from resource_management.core import sudo -from resource_management.core.logger import Logger -from resource_management.core.source import StaticFile from resource_management.libraries import XmlConfig +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.functions.default import default from resource_management.libraries.functions.format import format -from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions import StackFeature -from resource_management.libraries.functions.decorator import retry from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.script.script import Script -class Master(Script): +class Master(Script): def install(self, env): import params env.set_params(params) @@ -185,6 +185,45 @@ class Master(Script): group=params.zeppelin_group, mode=0644) + def check_and_copy_notebook_in_hdfs(self, params): + if params.config['configurations']['zeppelin-config']['zeppelin.notebook.dir'].startswith("/"): + notebook_directory = params.config['configurations']['zeppelin-config']['zeppelin.notebook.dir'] + else: + notebook_directory = "/user/" + format("{zeppelin_user}") + "/" + \ + params.config['configurations']['zeppelin-config']['zeppelin.notebook.dir'] + + kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) + kinit_if_needed = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};") + + notebook_directory_exists = shell.call(format("{kinit_if_needed} hdfs --config {hadoop_conf_dir} dfs -test -e {notebook_directory};echo $?"), + user=params.zeppelin_user)[1] + + #if there is no kerberos setup then the string will contain "-bash: kinit: command not found" + if "\n" in notebook_directory_exists: + notebook_directory_exists = notebook_directory_exists.split("\n")[1] + + # '1' means it does not exists + if notebook_directory_exists == '1': + # hdfs dfs -mkdir {notebook_directory} + params.HdfsResource(format("{notebook_directory}"), + type="directory", + action="create_on_execute", + owner=params.zeppelin_user, + recursive_chown=True, + recursive_chmod=True + ) + + # hdfs dfs -put /usr/hdp/current/zeppelin-server/notebook/ {notebook_directory} + params.HdfsResource(format("{notebook_directory}"), + type="directory", + action="create_on_execute", + source=params.notebook_dir, + owner=params.zeppelin_user, + recursive_chown=True, + recursive_chmod=True + ) + + def stop(self, env, upgrade_type=None): import params self.create_zeppelin_log_dir(env) @@ -202,6 +241,10 @@ class Master(Script): Execute(("chown", "-R", format("{zeppelin_user}") + ":" + format("{zeppelin_group}"), os.path.join(params.zeppelin_dir, "notebook")), sudo=True) + if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config'] \ + and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage'] == 'org.apache.zeppelin.notebook.repo.HdfsNotebookRepo': + self.check_and_copy_notebook_in_hdfs(params) + if params.security_enabled: zeppelin_kinit_cmd = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal}; ") Execute(zeppelin_kinit_cmd, user=params.zeppelin_user) http://git-wip-us.apache.org/repos/asf/ambari/blob/a679281f/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py index f5a2a37..3242f26 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py @@ -62,6 +62,8 @@ executor_mem = config['configurations']['zeppelin-env']['zeppelin.executor.mem'] executor_instances = config['configurations']['zeppelin-env'][ 'zeppelin.executor.instances'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] + spark_jar_dir = config['configurations']['zeppelin-env']['zeppelin.spark.jar.dir'] spark_jar = format("{spark_jar_dir}/zeppelin-spark-0.5.5-SNAPSHOT.jar") setup_view = True http://git-wip-us.apache.org/repos/asf/ambari/blob/a679281f/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml index 2bfc0dc..ca6b295 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml @@ -77,7 +77,7 @@ </property> <property> <name>zeppelin.notebook.storage</name> - <value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value> + <value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value> <description>notebook persistence layer implementation. If S3 is used, set this to org.apache.zeppelin.notebook.repo.S3NotebookRepo instead. If S3 is used to store the notebooks, it is necessary to use the following folder structure