Repository: ambari Updated Branches: refs/heads/trunk 9906c393a -> 4ddaed0eb
AMBARI-16031. Create "/hadoop/llap/local" on each host and disk in Kerberized cluster for LLAP (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4ddaed0e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4ddaed0e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4ddaed0e Branch: refs/heads/trunk Commit: 4ddaed0eb7ae617c85b7ddd97b84e3d47f4194bd Parents: 9906c39 Author: Alejandro Fernandez <[email protected]> Authored: Tue Apr 19 18:31:51 2016 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Fri Apr 22 15:23:43 2016 -0700 ---------------------------------------------------------------------- .../HIVE/0.12.0.2.0/kerberos.json | 5 ++ .../common-services/YARN/2.1.0.2.0/metainfo.xml | 10 ++++ .../2.1.0.2.0/package/scripts/nodemanager.py | 12 ++++- .../2.1.0.2.0/package/scripts/params_linux.py | 37 +++++++++++++-- .../YARN/2.1.0.2.0/package/scripts/yarn.py | 49 +++++++++++++++++++- .../stacks/HDP/2.5/services/HIVE/kerberos.json | 3 +- ambari-web/app/controllers/main/service/item.js | 38 +++++++++++++++ ambari-web/app/messages.js | 7 ++- ambari-web/app/models/host_component.js | 7 +++ ambari-web/app/views/main/service/item.js | 13 +++++- ambari-web/test/views/main/service/item_test.js | 2 + 11 files changed, 173 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json index 872bfac..a9e3840 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json @@ -18,6 +18,11 @@ } }, { + "hive-interactive-site": { + "hive.llap.daemon.work.dirs": "/hadoop/llap/local" + } + }, + { "webhcat-site": { "templeton.kerberos.secret": "secret", "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}" http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml index 10e6405..7034404 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml @@ -91,6 +91,16 @@ <!-- Used by decommission and recommission --> <masterComponent>RESOURCEMANAGER</masterComponent> </bulkCommands> + <customCommands> + <customCommand> + <name>CREATE_YARN_DIRECTORIES</name> + <commandScript> + <script>scripts/nodemanager.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </customCommand> + </customCommands> <logs> <log> <logId>yarn_nodemanager</logId> http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py index 6837c27..b1d6351 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py @@ -30,7 +30,7 @@ from resource_management.libraries.functions.format import format from resource_management.libraries.functions.security_commons import build_expectations, \ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ FILE_TYPE_XML -from yarn import yarn +from yarn import yarn, create_hive_llap_work_dir from service import service from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyImpl @@ -90,6 +90,16 @@ class NodemanagerDefault(Nodemanager): env.set_params(status_params) check_process_status(status_params.nodemanager_pid_file) + def create_yarn_directories(self, env): + """ + Custom command to Create Directories, e.g., needed by YARN Apps on Slider + """ + Logger.info("Custom Command to Create Directories") + import params + env.set_params(params) + create_hive_llap_work_dir(params) + + def security_status(self, env): import status_params env.set_params(status_params) http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 19dbb60..8fbefb2 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -19,6 +19,7 @@ Ambari Agent """ import os +import re from resource_management.libraries.script.script import Script from resource_management.libraries.resources.hdfs_resource import HdfsResource @@ -184,11 +185,39 @@ nm_address = config['configurations']['yarn-site']['yarn.nodemanager.address'] if hostname and nm_address and nm_address.startswith("0.0.0.0:"): nm_address = nm_address.replace("0.0.0.0", hostname) -nm_local_dirs = config['configurations']['yarn-site']['yarn.nodemanager.local-dirs'] -nm_log_dirs = config['configurations']['yarn-site']['yarn.nodemanager.log-dirs'] +# Initialize lists of work directories. +nm_local_dirs = default("/configurations/yarn-site/yarn.nodemanager.local-dirs", "") +nm_log_dirs = default("/configurations/yarn-site/yarn.nodemanager.log-dirs", "") -nm_local_dirs_list = nm_local_dirs.split(',') -nm_log_dirs_list = nm_log_dirs.split(',') +# If Hive Interactive Server is present, then default value will be ${yarn.nodemanager.local-dirs} +# During kerberization, kerberos.json changes the value to "/hadoop/llap/local". +# This is a comma separated list of directories. +hive_llap_work_dirs = default("/configurations/hive-interactive-site/hive.llap.daemon.work.dirs", "") + +# It can also be a dynamic variable, such as "${yarn.nodemanager.local-dirs}" +p = re.compile(r"\$\{(.*?)\}") +m = p.search(hive_llap_work_dirs) +if m and len(m.groups()) == 1: + hive_llap_work_dirs = default("/configurations/yarn-site/" + m.group(1), "") + +nm_local_dirs_list = nm_local_dirs.split(",") +nm_log_dirs_list = nm_log_dirs.split(",") +hive_llap_work_dirs_list = hive_llap_work_dirs.split(",") + + +def cleanse_list(l): + if l is None: + return None + l = [e.strip() for e in l if e.strip() != ""] + return l if len(l) > 0 else None + +nm_local_dirs_list = cleanse_list(nm_local_dirs_list) +nm_log_dirs_list = cleanse_list(nm_log_dirs_list) +hive_llap_work_dirs_list = cleanse_list(hive_llap_work_dirs_list) + +# Needed for LLAP to determine if should create Hive LLAP Work Dirs. +hive_server_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', []) +hive_server_interactive_host = hive_server_interactive_hosts[0] if len(hive_server_interactive_hosts) > 0 else None distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar" hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar" http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index 735d8cb..5d1be15 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -19,11 +19,25 @@ Ambari Agent """ -from resource_management import * +# Python Imports import os + +# Ambari Common and Resource Management Imports +from resource_management.libraries.script.script import Script +from resource_management.core.resources.service import ServiceConfig +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.is_empty import is_empty +from resource_management.core.resources.system import Directory +from resource_management.core.resources.system import File +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.core.source import InlineTemplate +from resource_management.core.source import Template +from resource_management.core.logger import Logger from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst +# Local Imports + @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) def yarn(name = None): @@ -113,6 +127,7 @@ def yarn(name = None): recursive_ownership = True, ) + #<editor-fold desc="Node Manager Section"> if name == "nodemanager": # First start after enabling/disabling security @@ -156,6 +171,9 @@ def yarn(name = None): recursive_mode_flags = {'f': 'a+rw', 'd': 'a+rwx'}, ) + create_hive_llap_work_dir(params) + #</editor-fold> + if params.yarn_nodemanager_recovery_dir: Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), owner=params.yarn_user, @@ -445,3 +463,32 @@ def yarn(name = None): owner=params.mapred_user, group=params.user_group ) + + +def create_hive_llap_work_dir(params): + """ + Create the work directory needed by LLAP, which is required on all NodeManagers. + This needs to be whenever NodeManagers are restarted, or after Hive Server Interactive and LLAP are added and started + via a custom command + :param params: Command parameters dictionary. + """ + if params.hive_llap_work_dirs_list is None or params.hive_server_interactive_host is None or not params.security_enabled: + Logger.info("Skip creating any Hive LLAP work dir since either it's empty, Hive Interactive is not present, " + "or cluster is not kerberized.") + return + + # If we already created this dir list because it has the same value as NM Local Dirs, then skip it + skip = False + if params.nm_local_dirs_list is not None and set(params.nm_local_dirs_list) == set(params.hive_llap_work_dirs_list): + skip = True + + if skip: + Logger.info(format("Skip creating Hive LLAP Work Dirs since it is equivalent to NM Local Dirs: {nm_local_dirs_list}")) + else: + Logger.info(format("Hive Server Interactive is present on the cluster, ensure that the Hive LLAP work dirs exist: {hive_llap_work_dirs_list}")) + Directory(params.hive_llap_work_dirs_list, + owner=params.yarn_user, + group=params.user_group, + create_parents=True, + ignore_failures=False, + mode=0775) http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json index a03a1b7..b5ec1c9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json @@ -31,7 +31,8 @@ { "hive-interactive-site": { "hadoop.security.authentication": "KERBEROS", - "hadoop.security.authorization": "true" + "hadoop.security.authorization": "true", + "hive.llap.daemon.work.dirs": "/hadoop/llap/local" } } ], http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-web/app/controllers/main/service/item.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/service/item.js b/ambari-web/app/controllers/main/service/item.js index 5707882..bfede8a 100644 --- a/ambari-web/app/controllers/main/service/item.js +++ b/ambari-web/app/controllers/main/service/item.js @@ -635,6 +635,44 @@ App.MainServiceItemController = Em.Controller.extend(App.SupportClientConfigsDow App.showAlertPopup(Em.I18n.t('common.error'), error); }, + createYARNDirectories: function(event) { + var context = Em.I18n.t('services.service.actions.run.createYARNDirectories'); + var command = "CREATE_YARN_DIRECTORIES"; + var component = "NODEMANAGER"; + var controller = this; + var hosts = App.Service.find('YARN').get('hostComponents').filterProperty('componentName', component).mapProperty('hostName'); + return App.showConfirmationPopup(function() { + App.ajax.send({ + name: 'service.item.executeCustomCommand', + sender: controller, + data: { + command: command, + context: context, + hosts: hosts.join(','), + serviceName: "YARN", + componentName: component + }, + success: 'createYARNDirectoriesSuccessCallback', + error: 'createYARNDirectoriesErrorCallback' + }); + }, Em.I18n.t('services.service.actions.run.createYARNDirectories.confirmation'), null, Em.I18n.t('popup.confirmation.commonHeader'), Em.I18n.t('ok'), false); + }, + createYARNDirectoriesSuccessCallback : function(data, ajaxOptions, params) { + if (data.Requests.id) { + App.router.get('backgroundOperationsController').showPopup(); + } + }, + createYARNDirectoriesErrorCallback : function(data) { + var error = Em.I18n.t('services.service.actions.run.executeCustomCommand.error'); + if (data && data.responseText) { + try { + var json = $.parseJSON(data.responseText); + error += json.message; + } catch (err) {} + } + App.showAlertPopup(Em.I18n.t('common.error'), error); + }, + /** * On click handler for rebalance Hdfs command from items menu */ http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-web/app/messages.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js index 6c0049d..907d9ae 100644 --- a/ambari-web/app/messages.js +++ b/ambari-web/app/messages.js @@ -1752,9 +1752,12 @@ Em.I18n.translations = { 'services.service.actions.run.stopLdapKnox.context':'Stop Demo LDAP', 'services.service.actions.run.startStopLdapKnox.error': 'Error during remote command: ', - // LLAP Custom Command + // Hive Server Interactive custom command to restart LLAP 'services.service.actions.run.restartLLAP':'Restart LLAP', - + // Node Manager custom command to refresh YARN Apps on Slider + 'services.service.actions.run.createYARNDirectories':'Create YARN Directories', + 'services.service.actions.run.createYARNDirectories.confirmation':'Please confirm. This will create necessary folders on the NodeManagers required for running YARN Apps on Slider, such as LLAP. This does not need to restart NodeManagers.', + 'services.service.actions.run.immediateStopHawqService.context':'Stop HAWQ Service (Immediate Mode)', 'services.service.actions.run.immediateStopHawqService.label':'Stop HAWQ Service (Immediate Mode)', 'services.service.actions.run.immediateStopHawqSegment.label':'Stop (Immediate Mode)', http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-web/app/models/host_component.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/models/host_component.js b/ambari-web/app/models/host_component.js index b4a369e..f10947f 100644 --- a/ambari-web/app/models/host_component.js +++ b/ambari-web/app/models/host_component.js @@ -370,6 +370,13 @@ App.HostComponentActionMap = { label: Em.I18n.t('services.service.actions.run.restartLLAP') + ' â', cssClass: 'icon-refresh' }, + CREATE_YARN_DIRECTORIES: { + action: 'createYARNDirectories', + customCommand: 'CREATE_YARN_DIRECTORIES', + context: Em.I18n.t('services.service.actions.run.createYARNDirectories'), + label: Em.I18n.t('services.service.actions.run.createYARNDirectories'), + cssClass: 'icon-refresh' + }, REBALANCEHDFS: { action: 'rebalanceHdfsNodes', customCommand: 'REBALANCEHDFS', http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-web/app/views/main/service/item.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/views/main/service/item.js b/ambari-web/app/views/main/service/item.js index 51f9bdd..36b3413 100644 --- a/ambari-web/app/views/main/service/item.js +++ b/ambari-web/app/views/main/service/item.js @@ -40,7 +40,8 @@ App.MainServiceItemView = Em.View.extend({ 'HAWQMASTER': ['IMMEDIATE_STOP_HAWQ_SERVICE', 'RUN_HAWQ_CHECK', 'HAWQ_CLEAR_CACHE', 'REMOVE_HAWQ_STANDBY', 'RESYNC_HAWQ_STANDBY'], 'HAWQSEGMENT': ['IMMEDIATE_STOP_HAWQ_SEGMENT'], 'HAWQSTANDBY': ['ACTIVATE_HAWQ_STANDBY'], - 'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"] + 'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"], + 'NODEMANAGER' : ["CREATE_YARN_DIRECTORIES"] }, addActionMap: function() { @@ -141,6 +142,16 @@ App.MainServiceItemView = Em.View.extend({ } if (this.get('serviceName') === 'YARN') { options.push(actionMap.REFRESHQUEUES); + var nodeManagerComponent = App.StackServiceComponent.find().findProperty('componentName', 'NODEMANAGER'); + var isNodeManagerPresent = allSlaves.contains('NODEMANAGER'); + if (nodeManagerComponent && isNodeManagerPresent) { + var nodeManagerCustomCommands = nodeManagerComponent.get('customCommands'); + nodeManagerCustomCommands.forEach(function (command) { + if (actionMap[command]) { + options.push(actionMap[command]); + } + }); + } } options.push(actionMap.RESTART_ALL); allSlaves.concat(allMasters).filter(function (_component) { http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-web/test/views/main/service/item_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/views/main/service/item_test.js b/ambari-web/test/views/main/service/item_test.js index 085eaad..f612f2d 100644 --- a/ambari-web/test/views/main/service/item_test.js +++ b/ambari-web/test/views/main/service/item_test.js @@ -494,6 +494,8 @@ describe('App.MainServiceItemView', function () { return Em.Object.create({ customCommands: ["DECOMMISSION", "REBALANCEHDFS"] }); case 'RESOURCEMANAGER': return Em.Object.create({ customCommands: ["DECOMMISSION", "REFRESHQUEUES"] }); + case 'NODEMANAGER': + return Em.Object.create({ customCommands: ["REFRESHQUEUES", "CREATE_YARN_DIRECTORIES"] }); case 'HBASE_MASTER': return Em.Object.create({ customCommands: ["DECOMMISSION"] }); case 'KNOX_GATEWAY':
