Repository: ambari Updated Branches: refs/heads/trunk 6ea7fd750 -> 9f61db3d1
AMBARI-15785. HiveServerInteractive. Kerberos Support. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9f61db3d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9f61db3d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9f61db3d Branch: refs/heads/trunk Commit: 9f61db3d12921dabf7572e708206b186dc8f5ffe Parents: 6ea7fd7 Author: sshridhar <[email protected]> Authored: Sun Apr 10 20:08:42 2016 -0700 Committer: swapan <[email protected]> Committed: Mon Apr 11 13:33:02 2016 -0700 ---------------------------------------------------------------------- .../package/scripts/hive_server_interactive.py | 48 +++- .../0.12.0.2.0/package/scripts/params_linux.py | 4 +- .../configuration/hive-interactive-site.xml | 16 ++ .../stacks/HDP/2.5/services/HIVE/kerberos.json | 125 ++++++++++ .../stacks/HDP/2.5/services/YARN/kerberos.json | 243 +++++++++++++++++++ .../stacks/HDP/2.5/services/stack_advisor.py | 9 + 6 files changed, 441 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py index 055296b..d909f6d 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py @@ -34,11 +34,13 @@ from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger from ambari_commons import OSCheck, OSConst from ambari_commons.os_family_impl import OsFamilyImpl +from pwd import getpwnam # Local Imports from setup_ranger_hive import setup_ranger_hive from hive_service_interactive import hive_service_interactive from hive_interactive import hive_interactive +from hive_server import HiveServerDefault class HiveServerInteractive(Script): @@ -74,6 +76,10 @@ class HiveServerDefault(HiveServerInteractive): env.set_params(params) self.configure(env) + if params.security_enabled: + # Do the security setup, internally calls do_kinit() + self.setup_security() + # TODO : We need have conditional [re]start of LLAP once "status check command" for LLAP is ready. # Check status and based on that decide on [re]starting. @@ -87,11 +93,15 @@ class HiveServerDefault(HiveServerInteractive): hive_service_interactive('hiveserver2', action='start', upgrade_type=upgrade_type) else: Logger.info("Skipping start of Hive Server Interactive due to LLAP start issue.") + raise Exception("Problem starting HiveServer2") def stop(self, env, upgrade_type=None): import params env.set_params(params) + if params.security_enabled: + self.do_kinit() + # TODO, why must Hive Server Interactive be stopped before LLAP??? # Stop Hive Interactive Server first @@ -114,7 +124,7 @@ class HiveServerDefault(HiveServerInteractive): pass def security_status(self, env): - pass + HiveServerDefault.security_status(env) def restart_llap(self, env): """ @@ -123,6 +133,10 @@ class HiveServerDefault(HiveServerInteractive): Logger.info("Custom Command to retart LLAP") import params env.set_params(params) + + if params.security_enabled: + self.do_kinit(); + self._llap_stop(env) self._llap_start(env) @@ -153,6 +167,9 @@ class HiveServerDefault(HiveServerInteractive): message += " " + error raise Fail(message) + """ + Controls the start of LLAP. + """ def _llap_start(self, env, cleanup=False): import params env.set_params(params) @@ -162,7 +179,14 @@ class HiveServerDefault(HiveServerInteractive): # TODO : Currently hardcoded the params. Need to read the suggested values from hive2/hive-site.xml. # TODO, ensure that script works as hive from cmd when not cd'ed in /homve/hive # Needs permission to write to hive home dir. - cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 -slider-am-container-mb {slider_am_container_mb} --loglevel INFO") + + cmd = '' + if params.security_enabled: + cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 -slider-am-container-mb " + "{slider_am_container_mb} --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab " + "{hive_llap_keytab_file} --slider-principal {hive_headless_keytab} --loglevel INFO") + else: + cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 -slider-am-container-mb {slider_am_container_mb} --loglevel INFO") run_file_path = None try: @@ -184,7 +208,6 @@ class HiveServerDefault(HiveServerInteractive): raise Fail("Did not find run.sh file in output: " + str(output)) Logger.info(format("Run file path: {run_file_path}")) - if os.path.isfile(run_file_path): Execute(run_file_path, user=params.hive_user) @@ -193,6 +216,7 @@ class HiveServerDefault(HiveServerInteractive): Logger.info("Sleeping for 30 secs") time.sleep(30) Logger.info("LLAP app deployed successfully.") + return True else: raise Fail(format("Did not find run file {run_file_path}")) except: @@ -207,7 +231,25 @@ class HiveServerDefault(HiveServerInteractive): # throw the original exception raise + return False + + """ + Does kinit and copies keytab for Hive/LLAP to HDFS. + """ + def setup_security(self): + import params + + self.do_kinit(); + + # Copy params.hive_llap_keytab_file to hdfs://<host>:<port>/user/<hive_user>/.slider/keytabs/<hive_user> , required by LLAP + slider_keytab_install_cmd = format("slider install-keytab --keytab {params.hive_llap_keytab_file} --folder {params.hive_user} --overwrite") + Execute(slider_keytab_install_cmd, user=params.hive_user) + + def do_kinit(self): + import params + hive_interactive_kinit_cmd = format("{kinit_path_local} -kt {hive_server2_keytab} {hive_principal}; ") + Execute(hive_interactive_kinit_cmd, user=params.hive_user) if __name__ == "__main__": HiveServerInteractive().execute() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index c7d57c0..f878141 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -498,7 +498,9 @@ if has_hive_interactive: # Tez for Hive interactive related tez_interactive_config_dir = os.path.realpath("/etc/tez_hive2/conf") tez_interactive_user = config['configurations']['tez-env']['tez_user'] - + if security_enabled: + hive_llap_keytab_file = config['configurations']['hive-interactive-site']['hive.llap.zk.sm.keytab.file'] + hive_headless_keytab = config['configurations']['hive-interactive-site']['hive.llap.zk.sm.principal'] # ranger host ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml index 5830a52..ba46f00 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml @@ -472,4 +472,20 @@ limitations under the License. <value>true</value> </property> + <property> + <name>hive.llap.daemon.work.dirs</name> + <value>${yarn.nodemanager.local-dirs}</value> + </property> + + <property> + <name>hive.llap.zk.sm.connectionString</name> + <value>localhost:2181</value> + <depends-on> + <property> + <type>zoo.cfg</type> + <name>clientPort</name> + </property> + </depends-on> + </property> + </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json new file mode 100644 index 0000000..fe0c2ec --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json @@ -0,0 +1,125 @@ +{ + "services": [ + { + "name": "HIVE", + "identities": [ + { + "name": "/spnego" + }, + { + "name": "/smokeuser" + } + ], + "configurations": [ + { + "hive-site": { + "hive.metastore.sasl.enabled": "true", + "hive.server2.authentication": "KERBEROS" + } + }, + { + "webhcat-site": { + "templeton.kerberos.secret": "secret", + "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}" + } + }, + { + "core-site": { + "hadoop.proxyuser.HTTP.hosts": "${clusterHostInfo/webhcat_server_host}" + } + }, + { + "hive-interactive-site": { + "hadoop.security.authentication": "KERBEROS", + "hadoop.security.authorization": "true" + } + } + ], + "components": [ + { + "name": "HIVE_METASTORE", + "identities": [ + { + "name": "/HIVE/HIVE_SEVER/hive_server_hive", + "principal": { + "configuration": "hive-site/hive.metastore.kerberos.principal" + }, + "keytab": { + "configuration": "hive-site/hive.metastore.kerberos.keytab.file" + } + } + ] + }, + { + "name": "HIVE_SERVER", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "hive_server_hive", + "principal": { + "value": "hive/_HOST@${realm}", + "type" : "service", + "configuration": "hive-site/hive.server2.authentication.kerberos.principal", + "local_username": "${hive-env/hive_user}" + }, + "keytab": { + "file": "${keytab_dir}/hive.service.keytab", + "owner": { + "name": "${hive-env/hive_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "r" + }, + "configuration": "hive-site/hive.server2.authentication.kerberos.keytab" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "hive-site/hive.server2.authentication.spnego.principal" + }, + "keytab": { + "configuration": "hive-site/hive.server2.authentication.spnego.keytab" + } + } + ] + }, + { + "name": "HIVE_SERVER_INTERACTIVE", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "/HIVE/HIVE_SERVER/hive_server_hive" + }, + { + "name": "/HIVE/HIVE_SERVER/spnego" + }, + { + "name": "/YARN/NODEMANAGER/llap_zk_hive" + } + ] + }, + { + "name": "WEBHCAT_SERVER", + "identities": [ + { + "name": "/spnego", + "principal": { + "configuration": "webhcat-site/templeton.kerberos.principal" + }, + "keytab": { + "configuration": "webhcat-site/templeton.kerberos.keytab" + } + } + ] + } + ] + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json new file mode 100644 index 0000000..367dfd8 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json @@ -0,0 +1,243 @@ +{ + "services": [ + { + "name": "YARN", + "identities": [ + { + "name": "/spnego" + }, + { + "name": "/smokeuser" + } + ], + "configurations": [ + { + "yarn-site": { + "yarn.timeline-service.enabled": "false", + "yarn.timeline-service.http-authentication.type": "kerberos", + "yarn.acl.enable": "true", + "yarn.timeline-service.http-authentication.signature.secret": "", + "yarn.timeline-service.http-authentication.signature.secret.file": "", + "yarn.timeline-service.http-authentication.signer.secret.provider": "", + "yarn.timeline-service.http-authentication.signer.secret.provider.object": "", + "yarn.timeline-service.http-authentication.token.validity": "", + "yarn.timeline-service.http-authentication.cookie.domain": "", + "yarn.timeline-service.http-authentication.cookie.path": "", + "yarn.timeline-service.http-authentication.proxyusers.*.hosts": "", + "yarn.timeline-service.http-authentication.proxyusers.*.users": "", + "yarn.timeline-service.http-authentication.proxyusers.*.groups": "", + "yarn.timeline-service.http-authentication.kerberos.name.rules": "", + "yarn.resourcemanager.proxyusers.*.groups": "", + "yarn.resourcemanager.proxyusers.*.hosts": "", + "yarn.resourcemanager.proxyusers.*.users": "", + "yarn.resourcemanager.proxy-user-privileges.enabled": "true", + "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "" + } + }, + { + "core-site": { + "hadoop.proxyuser.${yarn-env/yarn_user}.groups": "*", + "hadoop.proxyuser.${yarn-env/yarn_user}.hosts": "${clusterHostInfo/rm_host}" + } + } + ], + "components": [ + { + "name": "NODEMANAGER", + "identities": [ + { + "name": "nodemanager_nm", + "principal": { + "value": "nm/_HOST@${realm}", + "type" : "service", + "configuration": "yarn-site/yarn.nodemanager.principal", + "local_username": "${yarn-env/yarn_user}" + }, + "keytab": { + "file": "${keytab_dir}/nm.service.keytab", + "owner": { + "name": "${yarn-env/yarn_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "yarn-site/yarn.nodemanager.keytab" + } + }, + { + "name": "/HIVE/HIVE_SERVER/hive_server_hive", + "principal": { + "configuration": "hive-interactive-site/hive.llap.daemon.service.principal" + }, + "keytab": { + "configuration": "hive-interactive-site/hive.llap.daemon.keytab.file" + } + }, + { + "name": "llap_zk_hive", + "principal": { + "value": "hive@${realm}", + "type" : "user", + "configuration": "hive-interactive-site/hive.llap.zk.sm.principal" + }, + "keytab": { + "file": "${keytab_dir}/hive.llap.zk.sm.keytab", + "owner": { + "name": "${yarn-env/yarn_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "r" + }, + "configuration": "hive-interactive-site/hive.llap.zk.sm.keytab.file" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "yarn-site/yarn.nodemanager.webapp.spnego-principal" + }, + "keytab": { + "configuration": "yarn-site/yarn.nodemanager.webapp.spnego-keytab-file" + } + } + ], + "configurations": [ + { + "yarn-site": { + "yarn.nodemanager.container-executor.class": "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor" + } + } + ] + }, + { + "name": "RESOURCEMANAGER", + "identities": [ + { + "name": "resource_manager_rm", + "principal": { + "value": "rm/_HOST@${realm}", + "type" : "service", + "configuration": "yarn-site/yarn.resourcemanager.principal", + "local_username": "${yarn-env/yarn_user}" + }, + "keytab": { + "file": "${keytab_dir}/rm.service.keytab", + "owner": { + "name": "${yarn-env/yarn_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "yarn-site/yarn.resourcemanager.keytab" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "yarn-site/yarn.resourcemanager.webapp.spnego-principal" + }, + "keytab": { + "configuration": "yarn-site/yarn.resourcemanager.webapp.spnego-keytab-file" + } + } + ] + }, + { + "name": "APP_TIMELINE_SERVER", + "identities": [ + { + "name": "app_timeline_server_yarn", + "principal": { + "value": "yarn/_HOST@${realm}", + "type" : "service", + "configuration": "yarn-site/yarn.timeline-service.principal", + "local_username": "${yarn-env/yarn_user}" + }, + "keytab": { + "file": "${keytab_dir}/yarn.service.keytab", + "owner": { + "name": "${yarn-env/yarn_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "yarn-site/yarn.timeline-service.keytab" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "yarn-site/yarn.timeline-service.http-authentication.kerberos.principal" + }, + "keytab": { + "configuration": "yarn-site/yarn.timeline-service.http-authentication.kerberos.keytab" + } + }, + { + "name": "/HDFS/NAMENODE/hdfs" + } + ] + } + ] + }, + { + "name": "MAPREDUCE2", + "identities": [ + { + "name": "/spnego" + }, + { + "name": "/smokeuser" + } + ], + "components": [ + { + "name": "HISTORYSERVER", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + }, + { + "name": "history_server_jhs", + "principal": { + "value": "jhs/_HOST@${realm}", + "type" : "service", + "configuration": "mapred-site/mapreduce.jobhistory.principal", + "local_username": "${mapred-env/mapred_user}" + }, + "keytab": { + "file": "${keytab_dir}/jhs.service.keytab", + "owner": { + "name": "${mapred-env/mapred_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "mapred-site/mapreduce.jobhistory.keytab" + } + }, + { + "name": "/spnego", + "principal": { + "configuration": "mapred-site/mapreduce.jobhistory.webapp.spnego-principal" + }, + "keytab": { + "configuration": "mapred-site/mapreduce.jobhistory.webapp.spnego-keytab-file" + } + } + ] + } + ] + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 1812d05..7407da3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -121,6 +121,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor): pass pass + if 'hive-interactive-site' in services['configurations'] and \ + services['configurations']['hive-interactive-site']['properties']['hive.llap.zk.sm.connectionString']: + + # Fill the property 'hive.llap.zk.sm.connectionString' required by Hive Server Interactive (HiveServer2) + zookeeper_host_port = self.getZKHostPortString(services) + if zookeeper_host_port: + putHiveInteractiveSiteProperty = self.putProperty(configurations, "hive-interactive-site", services) + putHiveInteractiveSiteProperty("hive.llap.zk.sm.connectionString", zookeeper_host_port) + def recommendRangerConfigurations(self, configurations, clusterData, services, hosts): super(HDP25StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData, services, hosts)
