AMBARI-20200: hive-site.xml, hbase-site.xml, etc. are not found in class path for Zeppelin (Prabhjyot Singh via r-kamath)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b2f560ad Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b2f560ad Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b2f560ad Branch: refs/heads/branch-dev-logsearch Commit: b2f560adff7bf8c70e24c75e8e5c36da24ad97cc Parents: 0bbd545 Author: Renjith Kamath <[email protected]> Authored: Mon Mar 6 15:49:12 2017 +0530 Committer: Renjith Kamath <[email protected]> Committed: Mon Mar 6 15:49:12 2017 +0530 ---------------------------------------------------------------------- .../0.6.0.2.5/configuration/zeppelin-env.xml | 2 +- .../0.6.0.2.5/package/scripts/master.py | 36 +++++++++++++++----- .../0.6.0.2.5/package/scripts/params.py | 8 +++++ 3 files changed, 37 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b2f560ad/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml index 677158c..bdef319 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml @@ -89,7 +89,7 @@ export ZEPPELIN_PID_DIR={{zeppelin_pid_dir}} # export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). # export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth? # export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise - +export ZEPPELIN_INTP_CLASSPATH_OVERRIDES="{{external_dependency_conf}}" #### Spark interpreter configuration #### ## Use provided spark installation ## http://git-wip-us.apache.org/repos/asf/ambari/blob/b2f560ad/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py index 8a1fad6..fc24918 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py @@ -106,6 +106,17 @@ class Master(Script): mode=0755 ) + def create_zeppelin_hdfs_conf_dir(self, env): + import params + env.set_params(params) + Directory([params.external_dependency_conf], + owner=params.zeppelin_user, + group=params.zeppelin_group, + cd_access="a", + create_parents=True, + mode=0755 + ) + def chown_zeppelin_pid_dir(self, env): import params env.set_params(params) @@ -150,17 +161,26 @@ class Master(Script): File(format("{params.conf_dir}/log4j.properties"), content=params.log4j_properties_content, owner=params.zeppelin_user, group=params.zeppelin_group) + self.create_zeppelin_hdfs_conf_dir(env) # copy hive-site.xml only if Spark 1.x is installed - if 'spark-defaults' in params.config['configurations'] and \ - os.path.exists("/etc/spark/conf/hive-site.xml"): - File(format("{params.conf_dir}/hive-site.xml"), content=StaticFile("/etc/spark/conf/hive-site.xml"), - owner=params.zeppelin_user, group=params.zeppelin_group) + if 'spark-defaults' in params.config['configurations'] and params.is_hive_installed: + XmlConfig("hive-site.xml", + conf_dir=params.external_dependency_conf, + configurations=params.spark_hive_properties, + owner=params.zeppelin_user, + group=params.zeppelin_group, + mode=0644) - if len(params.hbase_master_hosts) > 0 and \ - os.path.exists("/etc/hbase/conf/hbase-site.xml"): + if len(params.hbase_master_hosts) > 0 and params.is_hbase_installed: # copy hbase-site.xml - File(format("{params.conf_dir}/hbase-site.xml"), content=StaticFile("/etc/hbase/conf/hbase-site.xml"), - owner=params.zeppelin_user, group=params.zeppelin_group) + XmlConfig("hbase-site.xml", + conf_dir=params.external_dependency_conf, + configurations=params.config['configurations']['hbase-site'], + configuration_attributes=params.config['configuration_attributes']['hbase-site'], + owner=params.zeppelin_user, + group=params.zeppelin_group, + mode=0644 + ) def stop(self, env, upgrade_type=None): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/b2f560ad/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py index 16a2782..9fd105b 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py @@ -95,6 +95,7 @@ zeppelin_hdfs_user_dir = format("/user/{zeppelin_user}") zeppelin_dir = os.path.join(*[install_dir, zeppelin_dirname]) conf_dir = "/etc/zeppelin/conf" +external_dependency_conf = "/etc/zeppelin/conf/external-dependency-conf" notebook_dir = os.path.join(*[install_dir, zeppelin_dirname, 'notebook']) # zeppelin-env.sh @@ -121,7 +122,12 @@ hive_metastore_port = None hive_server_port = None hive_zookeeper_quorum = None hive_server2_support_dynamic_service_discovery = None +is_hive_installed = False if 'hive_server_host' in master_configs and len(master_configs['hive_server_host']) != 0: + is_hive_installed = True + spark_hive_properties = { + 'hive.metastore.uris': config['configurations']['hive-site']['hive.metastore.uris'] + } hive_server_host = str(master_configs['hive_server_host'][0]) hive_metastore_host = str(master_configs['hive_metastore_host'][0]) hive_metastore_port = str( @@ -133,7 +139,9 @@ if 'hive_server_host' in master_configs and len(master_configs['hive_server_host # detect hbase details if installed zookeeper_znode_parent = None hbase_zookeeper_quorum = None +is_hbase_installed = False if 'hbase_master_hosts' in master_configs and 'hbase-site' in config['configurations']: + is_hbase_installed = True zookeeper_znode_parent = config['configurations']['hbase-site']['zookeeper.znode.parent'] hbase_zookeeper_quorum = config['configurations']['hbase-site']['hbase.zookeeper.quorum']
