Repository: ambari Updated Branches: refs/heads/branch-2.4 c41565852 -> 34abb2382
Revert "AMBARI-17983. (1). Revert AMBARI-16031 (Create /hadoop/llap/local on each host and disk in Kerberized cluster for LLAP), and (2). Remove the config 'hive.llap.daemon.work.dirs' as HIVE will manage the work directories itself." This reverts commit 773b128b8ecb8e464fbada13e88156555c1f21e2. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/34abb238 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/34abb238 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/34abb238 Branch: refs/heads/branch-2.4 Commit: 34abb238225e167eb714b45e3a67cfc9119ff3f3 Parents: c415658 Author: Swapan Shridhar <[email protected]> Authored: Wed Aug 3 16:52:14 2016 -0700 Committer: Swapan Shridhar <[email protected]> Committed: Wed Aug 3 16:52:55 2016 -0700 ---------------------------------------------------------------------- .../common-services/YARN/2.1.0.2.0/metainfo.xml | 10 ++++++ .../2.1.0.2.0/package/scripts/nodemanager.py | 10 ++++++ .../2.1.0.2.0/package/scripts/params_linux.py | 34 ++++++++++++++++-- .../YARN/2.1.0.2.0/package/scripts/yarn.py | 33 ++++++++++++++++- .../configuration/hive-interactive-site.xml | 10 ++++++ .../stacks/HDP/2.5/services/HIVE/kerberos.json | 5 +++ ambari-web/app/controllers/main/service/item.js | 38 ++++++++++++++++++++ ambari-web/app/messages.js | 3 ++ ambari-web/app/models/host_component.js | 7 ++++ ambari-web/app/views/main/service/item.js | 13 ++++++- ambari-web/test/views/main/service/item_test.js | 2 ++ 11 files changed, 160 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/34abb238/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml index f80fa8a..35329bc 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml @@ -91,6 +91,16 @@ <!-- Used by decommission and recommission --> <masterComponent>RESOURCEMANAGER</masterComponent> </bulkCommands> + <customCommands> + <customCommand> + <name>CREATE_YARN_DIRECTORIES</name> + <commandScript> + <script>scripts/nodemanager.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </customCommand> + </customCommands> <logs> <log> <logId>yarn_nodemanager</logId> http://git-wip-us.apache.org/repos/asf/ambari/blob/34abb238/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py index 6fcdf91..7be49d5 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py @@ -89,6 +89,16 @@ class NodemanagerDefault(Nodemanager): env.set_params(status_params) check_process_status(status_params.nodemanager_pid_file) + def create_yarn_directories(self, env): + """ + Custom command to Create Directories, e.g., needed by YARN Apps on Slider + """ + Logger.info("Custom Command to Create Directories") + import params + env.set_params(params) + create_hive_llap_work_dir(params) + + def security_status(self, env): import status_params env.set_params(status_params) http://git-wip-us.apache.org/repos/asf/ambari/blob/34abb238/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index b2c635d..154e433 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -19,6 +19,7 @@ Ambari Agent """ import os +import re from resource_management.libraries.script.script import Script from resource_management.libraries.resources.hdfs_resource import HdfsResource @@ -203,12 +204,39 @@ if hostname and nm_address and nm_address.startswith("0.0.0.0:"): nm_local_dirs = default("/configurations/yarn-site/yarn.nodemanager.local-dirs", "") nm_log_dirs = default("/configurations/yarn-site/yarn.nodemanager.log-dirs", "") -nm_local_dirs_list = nm_local_dirs.split(',') -nm_log_dirs_list = nm_log_dirs.split(',') - nm_log_dir_to_mount_file = "/var/lib/ambari-agent/data/yarn/yarn_log_dir_mount.hist" nm_local_dir_to_mount_file = "/var/lib/ambari-agent/data/yarn/yarn_local_dir_mount.hist" +# If Hive Interactive Server is present, then default value will be ${yarn.nodemanager.local-dirs} +# During kerberization, kerberos.json changes the value to "/hadoop/llap/local". +# This is a comma separated list of directories. +hive_llap_work_dirs = default("/configurations/hive-interactive-site/hive.llap.daemon.work.dirs", "") + +# It can also be a dynamic variable, such as "${yarn.nodemanager.local-dirs}" +p = re.compile(r"\$\{(.*?)\}") +m = p.search(hive_llap_work_dirs) +if m and len(m.groups()) == 1: + hive_llap_work_dirs = default("/configurations/yarn-site/" + m.group(1), "") + +nm_local_dirs_list = nm_local_dirs.split(",") +nm_log_dirs_list = nm_log_dirs.split(",") +hive_llap_work_dirs_list = hive_llap_work_dirs.split(",") + + +def cleanse_list(l): + if l is None: + return None + l = [e.strip() for e in l if e.strip() != ""] + return l if len(l) > 0 else None + +nm_local_dirs_list = cleanse_list(nm_local_dirs_list) +nm_log_dirs_list = cleanse_list(nm_log_dirs_list) +hive_llap_work_dirs_list = cleanse_list(hive_llap_work_dirs_list) + +# Needed for LLAP to determine if should create Hive LLAP Work Dirs. +hive_server_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', []) +hive_server_interactive_host = hive_server_interactive_hosts[0] if len(hive_server_interactive_hosts) > 0 else None + distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar" hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar" http://git-wip-us.apache.org/repos/asf/ambari/blob/34abb238/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index f5acb11..1b44b1b 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -202,6 +202,8 @@ def yarn(name=None, config_dir=None): mode=0644, content=nm_local_dir_to_mount_file_content ) + + create_hive_llap_work_dir(params) #</editor-fold> if params.yarn_nodemanager_recovery_dir: @@ -496,4 +498,33 @@ def yarn(name=None, config_dir=None): File(os.path.join(config_dir, 'ssl-server.xml.example'), owner=params.mapred_user, group=params.user_group - ) \ No newline at end of file + ) + + +def create_hive_llap_work_dir(params): + """ + Create the work directory needed by LLAP, which is required on all NodeManagers. + This needs to be whenever NodeManagers are restarted, or after Hive Server Interactive and LLAP are added and started + via a custom command + :param params: Command parameters dictionary. + """ + if params.hive_llap_work_dirs_list is None or params.hive_server_interactive_host is None or not params.security_enabled: + Logger.info("Skip creating any Hive LLAP work dir since either it's empty, Hive Interactive is not present, " + "or cluster is not kerberized.") + return + + # If we already created this dir list because it has the same value as NM Local Dirs, then skip it + skip = False + if params.nm_local_dirs_list is not None and set(params.nm_local_dirs_list) == set(params.hive_llap_work_dirs_list): + skip = True + + if skip: + Logger.info(format("Skip creating Hive LLAP Work Dirs since it is equivalent to NM Local Dirs: {nm_local_dirs_list}")) + else: + Logger.info(format("Hive Server Interactive is present on the cluster, ensure that the Hive LLAP work dirs exist: {hive_llap_work_dirs_list}")) + Directory(params.hive_llap_work_dirs_list, + owner=params.yarn_user, + group=params.user_group, + create_parents=True, + ignore_failures=False, + mode=0775) http://git-wip-us.apache.org/repos/asf/ambari/blob/34abb238/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml index 14ab470..776c2b6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml @@ -607,6 +607,16 @@ limitations under the License. <on-ambari-upgrade add="true"/> </property> <property> + <name>hive.llap.daemon.work.dirs</name> + <value>${yarn.nodemanager.local-dirs}</value> + <description> + Working directories for the daemon. Needs to be set for a secure cluster, since LLAP may + not have access to the default YARN working directories. yarn.nodemanager.local-dirs is + used if this is not set + </description> + <on-ambari-upgrade add="true"/> + </property> + <property> <name>hive.llap.zk.sm.connectionString</name> <value>localhost:2181</value> <description>ZooKeeper connection string for ZooKeeper SecretManager.</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/34abb238/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json index 37a7e82..4fae665 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json @@ -29,6 +29,11 @@ } }, { + "hive-interactive-site": { + "hive.llap.daemon.work.dirs": "/hadoop/llap/local" + } + }, + { "ranger-hive-audit": { "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule", "xasecure.audit.jaas.Client.loginModuleControlFlag": "required", http://git-wip-us.apache.org/repos/asf/ambari/blob/34abb238/ambari-web/app/controllers/main/service/item.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/service/item.js b/ambari-web/app/controllers/main/service/item.js index 6ce811d..e3df70b 100644 --- a/ambari-web/app/controllers/main/service/item.js +++ b/ambari-web/app/controllers/main/service/item.js @@ -693,6 +693,44 @@ App.MainServiceItemController = Em.Controller.extend(App.SupportClientConfigsDow App.showAlertPopup(Em.I18n.t('common.error'), error); }, + createYARNDirectories: function(event) { + var context = Em.I18n.t('services.service.actions.run.createYARNDirectories'); + var command = "CREATE_YARN_DIRECTORIES"; + var component = "NODEMANAGER"; + var controller = this; + var hosts = App.Service.find('YARN').get('hostComponents').filterProperty('componentName', component).mapProperty('hostName'); + return App.showConfirmationPopup(function() { + App.ajax.send({ + name: 'service.item.executeCustomCommand', + sender: controller, + data: { + command: command, + context: context, + hosts: hosts.join(','), + serviceName: "YARN", + componentName: component + }, + success: 'createYARNDirectoriesSuccessCallback', + error: 'createYARNDirectoriesErrorCallback' + }); + }, Em.I18n.t('services.service.actions.run.createYARNDirectories.confirmation'), null, Em.I18n.t('popup.confirmation.commonHeader'), Em.I18n.t('ok'), false); + }, + createYARNDirectoriesSuccessCallback : function(data, ajaxOptions, params) { + if (data.Requests.id) { + App.router.get('backgroundOperationsController').showPopup(); + } + }, + createYARNDirectoriesErrorCallback : function(data) { + var error = Em.I18n.t('services.service.actions.run.executeCustomCommand.error'); + if (data && data.responseText) { + try { + var json = $.parseJSON(data.responseText); + error += json.message; + } catch (err) {} + } + App.showAlertPopup(Em.I18n.t('common.error'), error); + }, + /** * On click handler for rebalance Hdfs command from items menu */ http://git-wip-us.apache.org/repos/asf/ambari/blob/34abb238/ambari-web/app/messages.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js index febb2ad..489e9ae 100644 --- a/ambari-web/app/messages.js +++ b/ambari-web/app/messages.js @@ -1816,6 +1816,9 @@ Em.I18n.translations = { // Hive Server Interactive custom command to restart LLAP 'services.service.actions.run.restartLLAP':'Restart LLAP', + // Node Manager custom command to refresh YARN Apps on Slider + 'services.service.actions.run.createYARNDirectories':'Create YARN Directories', + 'services.service.actions.run.createYARNDirectories.confirmation':'Please confirm. This will create necessary folders on the NodeManagers required for running YARN Apps on Slider, such as LLAP. This does not need to restart NodeManagers.', 'services.service.actions.run.immediateStopHawqService.context':'Stop HAWQ Service (Immediate Mode)', 'services.service.actions.run.immediateStopHawqService.label':'Stop HAWQ Service (Immediate Mode)', 'services.service.actions.run.immediateStopHawqSegment.label':'Stop (Immediate Mode)', http://git-wip-us.apache.org/repos/asf/ambari/blob/34abb238/ambari-web/app/models/host_component.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/models/host_component.js b/ambari-web/app/models/host_component.js index dca7ff0..91642c9 100644 --- a/ambari-web/app/models/host_component.js +++ b/ambari-web/app/models/host_component.js @@ -372,6 +372,13 @@ App.HostComponentActionMap = { label: Em.I18n.t('services.service.actions.run.restartLLAP') + ' â', cssClass: 'icon-refresh' }, + CREATE_YARN_DIRECTORIES: { + action: 'createYARNDirectories', + customCommand: 'CREATE_YARN_DIRECTORIES', + context: Em.I18n.t('services.service.actions.run.createYARNDirectories'), + label: Em.I18n.t('services.service.actions.run.createYARNDirectories'), + cssClass: 'icon-refresh' + }, REBALANCEHDFS: { action: 'rebalanceHdfsNodes', customCommand: 'REBALANCEHDFS', http://git-wip-us.apache.org/repos/asf/ambari/blob/34abb238/ambari-web/app/views/main/service/item.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/views/main/service/item.js b/ambari-web/app/views/main/service/item.js index a007e17..02ac426 100644 --- a/ambari-web/app/views/main/service/item.js +++ b/ambari-web/app/views/main/service/item.js @@ -40,7 +40,8 @@ App.MainServiceItemView = Em.View.extend({ 'HAWQMASTER': ['IMMEDIATE_STOP_HAWQ_SERVICE', 'RUN_HAWQ_CHECK', 'HAWQ_CLEAR_CACHE', 'REMOVE_HAWQ_STANDBY', 'RESYNC_HAWQ_STANDBY'], 'HAWQSEGMENT': ['IMMEDIATE_STOP_HAWQ_SEGMENT'], 'HAWQSTANDBY': ['ACTIVATE_HAWQ_STANDBY'], - 'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"] + 'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"], + 'NODEMANAGER' : ["CREATE_YARN_DIRECTORIES"] }, addActionMap: function() { @@ -141,6 +142,16 @@ App.MainServiceItemView = Em.View.extend({ } if (this.get('serviceName') === 'YARN') { options.push(actionMap.REFRESHQUEUES); + var nodeManagerComponent = App.StackServiceComponent.find().findProperty('componentName', 'NODEMANAGER'); + var isNodeManagerPresent = allSlaves.contains('NODEMANAGER'); + if (nodeManagerComponent && isNodeManagerPresent) { + var nodeManagerCustomCommands = nodeManagerComponent.get('customCommands'); + nodeManagerCustomCommands.forEach(function (command) { + if (actionMap[command]) { + options.push(actionMap[command]); + } + }); + } } options.push(actionMap.RESTART_ALL); allSlaves.concat(allMasters).filter(function (_component) { http://git-wip-us.apache.org/repos/asf/ambari/blob/34abb238/ambari-web/test/views/main/service/item_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/views/main/service/item_test.js b/ambari-web/test/views/main/service/item_test.js index f318185..ecfa7de 100644 --- a/ambari-web/test/views/main/service/item_test.js +++ b/ambari-web/test/views/main/service/item_test.js @@ -494,6 +494,8 @@ describe('App.MainServiceItemView', function () { return Em.Object.create({ customCommands: ["DECOMMISSION", "REBALANCEHDFS"] }); case 'RESOURCEMANAGER': return Em.Object.create({ customCommands: ["DECOMMISSION", "REFRESHQUEUES"] }); + case 'NODEMANAGER': + return Em.Object.create({ customCommands: ["REFRESHQUEUES", "CREATE_YARN_DIRECTORIES"] }); case 'HBASE_MASTER': return Em.Object.create({ customCommands: ["DECOMMISSION"] }); case 'KNOX_GATEWAY':
