Repository: ambari Updated Branches: refs/heads/branch-2.4 8f76a5fb1 -> 045521f7a
AMBARI-17321 Zeppelin: Kerberos configurations and latest interpreter config missing. (Renjith Kamath via dipayanb) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/045521f7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/045521f7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/045521f7 Branch: refs/heads/branch-2.4 Commit: 045521f7ad83eab5728c441c5852ed28502b26b8 Parents: 8f76a5f Author: Dipayan Bhowmick <[email protected]> Authored: Wed Jun 22 00:33:39 2016 +0530 Committer: Dipayan Bhowmick <[email protected]> Committed: Wed Jun 22 00:33:39 2016 +0530 ---------------------------------------------------------------------- .../0.6.0.2.5/configuration/zeppelin-env.xml | 17 +++++++++ .../ZEPPELIN/0.6.0.2.5/kerberos.json | 29 ++++++++++++-- .../ZEPPELIN/0.6.0.2.5/metainfo.xml | 16 ++++++++ .../0.6.0.2.5/package/scripts/master.py | 40 ++++++++++++++++---- .../0.6.0.2.5/package/scripts/params.py | 12 +++++- .../0.6.0.2.5/package/scripts/service_check.py | 4 ++ 6 files changed, 105 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/045521f7/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml index a912272..844ae98 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml @@ -221,4 +221,21 @@ log4j.appender.dailyfile.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m </description> <on-ambari-upgrade add="true"/> </property> + + <property> + <name>zeppelin.server.kerberos.principal</name> + <value>none</value> + <description> + Kerberos principal name for the Zeppelin. + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>zeppelin.server.kerberos.keytab</name> + <value>none</value> + <description> + Location of the kerberos keytab file for the Zeppelin. + </description> + <on-ambari-upgrade add="true"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/045521f7/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/kerberos.json b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/kerberos.json index f69ae45..3dcc67c 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/kerberos.json +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/kerberos.json @@ -5,13 +5,36 @@ "identities": [ { "name": "/smokeuser" + }, + { + "name": "zeppelin_user", + "principal": { + "value": "zeppelin/${cluster_name}@${realm}", + "type" : "user", + "configuration": "zeppelin-env/zeppelin.server.kerberos.principal", + "local_username" : "zeppelin" + }, + "keytab": { + "file": "${keytab_dir}/zeppelin.server.kerberos.keytab", + "owner": { + "name": "zeppelin", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "zeppelin-env/zeppelin.server.kerberos.keytab" + } } ], - "components": [ + "configurations": [ { - "name": "ZEPPELIN_MASTER" + "zeppelin-env": { + "zeppelin.kerberos.enabled": "true" + } } ] } ] -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ambari/blob/045521f7/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml index f609156..c40a593 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml @@ -37,6 +37,22 @@ limitations under the License. <scriptType>PYTHON</scriptType> <timeout>10000</timeout> </commandScript> + <dependencies> + <dependency> + <name>SPARK/LIVY_SERVER</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>SPARK/SPARK_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> </component> </components> http://git-wip-us.apache.org/repos/asf/ambari/blob/045521f7/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py index 22d66b0..f2899d0 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py @@ -144,14 +144,27 @@ class Master(Script): def start(self, env): import params import status_params + import time self.configure(env) + if params.security_enabled: + spark_kinit_cmd = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal}; ") + Execute(spark_kinit_cmd, user=params.zeppelin_user) + if glob.glob( params.zeppelin_dir + '/interpreter/spark/dep/zeppelin-spark-dependencies-*.jar') and os.path.exists( glob.glob(params.zeppelin_dir + '/interpreter/spark/dep/zeppelin-spark-dependencies-*.jar')[0]): self.create_zeppelin_dir(params) - Execute(params.zeppelin_dir + '/bin/zeppelin-daemon.sh start >> ' + # if first_setup: + if not glob.glob(params.conf_dir + "/interpreter.json") and \ + not os.path.exists(params.conf_dir + "/interpreter.json"): + Execute(params.zeppelin_dir + '/bin/zeppelin-daemon.sh start >> ' + + params.zeppelin_log_file, user=params.zeppelin_user) + time.sleep(20) + self.update_zeppelin_interpreter() + + Execute(params.zeppelin_dir + '/bin/zeppelin-daemon.sh restart >> ' + params.zeppelin_log_file, user=params.zeppelin_user) pidfile = glob.glob(status_params.zeppelin_pid_dir + '/zeppelin-' + params.zeppelin_user + '*.pid')[0] @@ -159,11 +172,6 @@ class Master(Script): contents = open(pidfile).read() Execute('echo pid is ' + contents, user=params.zeppelin_user) - # if first_setup: - import time - time.sleep(20) - self.update_zeppelin_interpreter() - def status(self, env): import status_params env.set_params(status_params) @@ -187,15 +195,31 @@ class Master(Script): for notebooks in interpreter_settings: notebook = interpreter_settings[notebooks] - if notebook['group'] == 'hive' and params.hive_server_host: - notebook['properties']['hive.hiveserver2.url'] = 'jdbc:hive2://' +\ + if notebook['group'] == 'jdbc' and params.hive_server_host: + notebook['properties']['default.url'] = 'jdbc:hive2://' +\ params.hive_server_host +\ ':' + params.hive_server_port + notebook['properties']['default.driver'] = "org.apache.hive.jdbc.HiveDriver" + notebook['dependencies'] = [] + notebook['dependencies'].append( + {"groupArtifactVersion": "org.apache.hive:hive-jdbc:2.0.1", "local": "false"}) + notebook['dependencies'].append( + {"groupArtifactVersion": "org.apache.hadoop:hadoop-common:2.7.2", "local": "false"}) elif notebook['group'] == 'phoenix' and params.zookeeper_znode_parent \ and params.hbase_zookeeper_quorum: notebook['properties']['phoenix.jdbc.url'] = "jdbc:phoenix:" +\ params.hbase_zookeeper_quorum + ':' +\ params.zookeeper_znode_parent + elif notebook['group'] == 'livy' and params.livy_livyserver_host: + notebook['properties']['livy.spark.master'] = "yarn-cluster" + notebook['properties']['zeppelin.livy.principal'] = params.zeppelin_kerberos_principal + notebook['properties']['zeppelin.livy.keytab'] = params.zeppelin_kerberos_keytab + notebook['properties']['zeppelin.livy.url'] = "http://"; + params.livy_livyserver_host +\ + ":" + params.livy_livyserver_port + elif notebook['group'] == 'spark': + notebook['properties']['master'] = "yarn-cluster" + notebook['properties']['spark.yarn.principal'] = params.zeppelin_kerberos_principal + notebook['properties']['spark.yarn.keytab'] = params.zeppelin_kerberos_keytab interpreter_config_file = open(interpreter_config, "w+") interpreter_config_file.write(json.dumps(config_data, indent=2)) http://git-wip-us.apache.org/repos/asf/ambari/blob/045521f7/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py index e5ce9f2..770f8f5 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py @@ -21,16 +21,17 @@ limitations under the License. import functools import os import re - +from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.format import format from resource_management.libraries.functions.get_stack_version import get_stack_version +from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.script.script import Script -from resource_management.libraries.functions.format import format def get_port_from_url(address): if not (address is None): @@ -123,6 +124,9 @@ if 'spark.yarn.queue' in config['configurations']['spark-defaults']: else: spark_queue = 'default' +zeppelin_kerberos_keytab = config['configurations']['zeppelin-env']['zeppelin.server.kerberos.keytab'] +zeppelin_kerberos_principal = config['configurations']['zeppelin-env']['zeppelin.server.kerberos.principal'] + # e.g. 2.3 stack_version_unformatted = config['hostLevelParams']['stack_version'] @@ -134,6 +138,10 @@ full_stack_version = default("/commandParams/version", None) spark_client_version = get_stack_version('spark-client') +if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stack_version_formatted): + livy_livyserver_host = str(default("/clusterHostInfo/livy_server_hosts", [])[0]) + livy_livyserver_port = config['configurations']['livy-conf']['livy.server.port'] + hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] security_enabled = config['configurations']['cluster-env']['security_enabled'] hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] http://git-wip-us.apache.org/repos/asf/ambari/blob/045521f7/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/service_check.py index 5dfdcad..423444d 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/service_check.py @@ -26,6 +26,10 @@ class ZeppelinServiceCheck(Script): import params env.set_params(params) + if params.security_enabled: + spark_kinit_cmd = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal}; ") + Execute(spark_kinit_cmd, user=params.zeppelin_user) + Execute(format("curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k {zeppelin_host}:{zeppelin_port} | grep 200"), tries = 10, try_sleep=3,
