Repository: ambari Updated Branches: refs/heads/trunk 9bb772392 -> da1701de9
AMBARI-17983. (1). Revert AMBARI-16031 (Create /hadoop/llap/local on each host and disk in Kerberized cluster for LLAP), and (2). Remove the config 'hive.llap.daemon.work.dirs' as HIVE will manage the work directories itself. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/da1701de Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/da1701de Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/da1701de Branch: refs/heads/trunk Commit: da1701de99e55997e39e0f644c14e7ff2f927fb7 Parents: 9bb7723 Author: Swapan Shridhar <[email protected]> Authored: Mon Aug 1 16:04:40 2016 -0700 Committer: Swapan Shridhar <[email protected]> Committed: Wed Aug 3 12:31:22 2016 -0700 ---------------------------------------------------------------------- .../common-services/YARN/2.1.0.2.0/metainfo.xml | 10 ------ .../2.1.0.2.0/package/scripts/nodemanager.py | 12 +------ .../2.1.0.2.0/package/scripts/params_linux.py | 34 ++---------------- .../YARN/2.1.0.2.0/package/scripts/yarn.py | 33 +---------------- .../configuration/hive-interactive-site.xml | 10 ------ .../stacks/HDP/2.5/services/HIVE/kerberos.json | 5 --- ambari-web/app/controllers/main/service/item.js | 38 -------------------- ambari-web/app/messages.js | 3 -- ambari-web/app/models/host_component.js | 7 ---- ambari-web/app/views/main/service/item.js | 13 +------ ambari-web/test/views/main/service/item_test.js | 2 -- 11 files changed, 6 insertions(+), 161 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml index 35329bc..f80fa8a 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml @@ -91,16 +91,6 @@ <!-- Used by decommission and recommission --> <masterComponent>RESOURCEMANAGER</masterComponent> </bulkCommands> - <customCommands> - <customCommand> - <name>CREATE_YARN_DIRECTORIES</name> - <commandScript> - <script>scripts/nodemanager.py</script> - <scriptType>PYTHON</scriptType> - <timeout>600</timeout> - </commandScript> - </customCommand> - </customCommands> <logs> <log> <logId>yarn_nodemanager</logId> http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py index 106b734..bc8947a 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py @@ -31,7 +31,7 @@ from resource_management.libraries.functions.security_commons import build_expec cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ FILE_TYPE_XML from resource_management.core.logger import Logger -from yarn import yarn, create_hive_llap_work_dir +from yarn import yarn from service import service from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyImpl @@ -90,16 +90,6 @@ class NodemanagerDefault(Nodemanager): env.set_params(status_params) check_process_status(status_params.nodemanager_pid_file) - def create_yarn_directories(self, env): - """ - Custom command to Create Directories, e.g., needed by YARN Apps on Slider - """ - Logger.info("Custom Command to Create Directories") - import params - env.set_params(params) - create_hive_llap_work_dir(params) - - def security_status(self, env): import status_params env.set_params(status_params) http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 154e433..b2c635d 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -19,7 +19,6 @@ Ambari Agent """ import os -import re from resource_management.libraries.script.script import Script from resource_management.libraries.resources.hdfs_resource import HdfsResource @@ -204,39 +203,12 @@ if hostname and nm_address and nm_address.startswith("0.0.0.0:"): nm_local_dirs = default("/configurations/yarn-site/yarn.nodemanager.local-dirs", "") nm_log_dirs = default("/configurations/yarn-site/yarn.nodemanager.log-dirs", "") +nm_local_dirs_list = nm_local_dirs.split(',') +nm_log_dirs_list = nm_log_dirs.split(',') + nm_log_dir_to_mount_file = "/var/lib/ambari-agent/data/yarn/yarn_log_dir_mount.hist" nm_local_dir_to_mount_file = "/var/lib/ambari-agent/data/yarn/yarn_local_dir_mount.hist" -# If Hive Interactive Server is present, then default value will be ${yarn.nodemanager.local-dirs} -# During kerberization, kerberos.json changes the value to "/hadoop/llap/local". -# This is a comma separated list of directories. -hive_llap_work_dirs = default("/configurations/hive-interactive-site/hive.llap.daemon.work.dirs", "") - -# It can also be a dynamic variable, such as "${yarn.nodemanager.local-dirs}" -p = re.compile(r"\$\{(.*?)\}") -m = p.search(hive_llap_work_dirs) -if m and len(m.groups()) == 1: - hive_llap_work_dirs = default("/configurations/yarn-site/" + m.group(1), "") - -nm_local_dirs_list = nm_local_dirs.split(",") -nm_log_dirs_list = nm_log_dirs.split(",") -hive_llap_work_dirs_list = hive_llap_work_dirs.split(",") - - -def cleanse_list(l): - if l is None: - return None - l = [e.strip() for e in l if e.strip() != ""] - return l if len(l) > 0 else None - -nm_local_dirs_list = cleanse_list(nm_local_dirs_list) -nm_log_dirs_list = cleanse_list(nm_log_dirs_list) -hive_llap_work_dirs_list = cleanse_list(hive_llap_work_dirs_list) - -# Needed for LLAP to determine if should create Hive LLAP Work Dirs. -hive_server_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', []) -hive_server_interactive_host = hive_server_interactive_hosts[0] if len(hive_server_interactive_hosts) > 0 else None - distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar" hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar" http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py index cf5736e..93c3aaa 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py @@ -201,8 +201,6 @@ def yarn(name=None, config_dir=None): mode=0644, content=nm_local_dir_to_mount_file_content ) - - create_hive_llap_work_dir(params) #</editor-fold> if params.yarn_nodemanager_recovery_dir: @@ -497,33 +495,4 @@ def yarn(name=None, config_dir=None): File(os.path.join(config_dir, 'ssl-server.xml.example'), owner=params.mapred_user, group=params.user_group - ) - - -def create_hive_llap_work_dir(params): - """ - Create the work directory needed by LLAP, which is required on all NodeManagers. - This needs to be whenever NodeManagers are restarted, or after Hive Server Interactive and LLAP are added and started - via a custom command - :param params: Command parameters dictionary. - """ - if params.hive_llap_work_dirs_list is None or params.hive_server_interactive_host is None or not params.security_enabled: - Logger.info("Skip creating any Hive LLAP work dir since either it's empty, Hive Interactive is not present, " - "or cluster is not kerberized.") - return - - # If we already created this dir list because it has the same value as NM Local Dirs, then skip it - skip = False - if params.nm_local_dirs_list is not None and set(params.nm_local_dirs_list) == set(params.hive_llap_work_dirs_list): - skip = True - - if skip: - Logger.info(format("Skip creating Hive LLAP Work Dirs since it is equivalent to NM Local Dirs: {nm_local_dirs_list}")) - else: - Logger.info(format("Hive Server Interactive is present on the cluster, ensure that the Hive LLAP work dirs exist: {hive_llap_work_dirs_list}")) - Directory(params.hive_llap_work_dirs_list, - owner=params.yarn_user, - group=params.user_group, - create_parents=True, - ignore_failures=False, - mode=0775) + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml index 776c2b6..14ab470 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml @@ -607,16 +607,6 @@ limitations under the License. <on-ambari-upgrade add="true"/> </property> <property> - <name>hive.llap.daemon.work.dirs</name> - <value>${yarn.nodemanager.local-dirs}</value> - <description> - Working directories for the daemon. Needs to be set for a secure cluster, since LLAP may - not have access to the default YARN working directories. yarn.nodemanager.local-dirs is - used if this is not set - </description> - <on-ambari-upgrade add="true"/> - </property> - <property> <name>hive.llap.zk.sm.connectionString</name> <value>localhost:2181</value> <description>ZooKeeper connection string for ZooKeeper SecretManager.</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json index 4fae665..37a7e82 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json @@ -29,11 +29,6 @@ } }, { - "hive-interactive-site": { - "hive.llap.daemon.work.dirs": "/hadoop/llap/local" - } - }, - { "ranger-hive-audit": { "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule", "xasecure.audit.jaas.Client.loginModuleControlFlag": "required", http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-web/app/controllers/main/service/item.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/service/item.js b/ambari-web/app/controllers/main/service/item.js index e3df70b..6ce811d 100644 --- a/ambari-web/app/controllers/main/service/item.js +++ b/ambari-web/app/controllers/main/service/item.js @@ -693,44 +693,6 @@ App.MainServiceItemController = Em.Controller.extend(App.SupportClientConfigsDow App.showAlertPopup(Em.I18n.t('common.error'), error); }, - createYARNDirectories: function(event) { - var context = Em.I18n.t('services.service.actions.run.createYARNDirectories'); - var command = "CREATE_YARN_DIRECTORIES"; - var component = "NODEMANAGER"; - var controller = this; - var hosts = App.Service.find('YARN').get('hostComponents').filterProperty('componentName', component).mapProperty('hostName'); - return App.showConfirmationPopup(function() { - App.ajax.send({ - name: 'service.item.executeCustomCommand', - sender: controller, - data: { - command: command, - context: context, - hosts: hosts.join(','), - serviceName: "YARN", - componentName: component - }, - success: 'createYARNDirectoriesSuccessCallback', - error: 'createYARNDirectoriesErrorCallback' - }); - }, Em.I18n.t('services.service.actions.run.createYARNDirectories.confirmation'), null, Em.I18n.t('popup.confirmation.commonHeader'), Em.I18n.t('ok'), false); - }, - createYARNDirectoriesSuccessCallback : function(data, ajaxOptions, params) { - if (data.Requests.id) { - App.router.get('backgroundOperationsController').showPopup(); - } - }, - createYARNDirectoriesErrorCallback : function(data) { - var error = Em.I18n.t('services.service.actions.run.executeCustomCommand.error'); - if (data && data.responseText) { - try { - var json = $.parseJSON(data.responseText); - error += json.message; - } catch (err) {} - } - App.showAlertPopup(Em.I18n.t('common.error'), error); - }, - /** * On click handler for rebalance Hdfs command from items menu */ http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-web/app/messages.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js index 2d7ed6e..1f42685 100644 --- a/ambari-web/app/messages.js +++ b/ambari-web/app/messages.js @@ -1847,9 +1847,6 @@ Em.I18n.translations = { // Hive Server Interactive custom command to restart LLAP 'services.service.actions.run.restartLLAP':'Restart LLAP', - // Node Manager custom command to refresh YARN Apps on Slider - 'services.service.actions.run.createYARNDirectories':'Create YARN Directories', - 'services.service.actions.run.createYARNDirectories.confirmation':'Please confirm. This will create necessary folders on the NodeManagers required for running YARN Apps on Slider, such as LLAP. This does not need to restart NodeManagers.', 'services.service.actions.run.immediateStopHawqService.context':'Stop HAWQ Service (Immediate Mode)', 'services.service.actions.run.immediateStopHawqService.label':'Stop HAWQ Service (Immediate Mode)', 'services.service.actions.run.immediateStopHawqSegment.label':'Stop (Immediate Mode)', http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-web/app/models/host_component.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/models/host_component.js b/ambari-web/app/models/host_component.js index 91642c9..dca7ff0 100644 --- a/ambari-web/app/models/host_component.js +++ b/ambari-web/app/models/host_component.js @@ -372,13 +372,6 @@ App.HostComponentActionMap = { label: Em.I18n.t('services.service.actions.run.restartLLAP') + ' â', cssClass: 'icon-refresh' }, - CREATE_YARN_DIRECTORIES: { - action: 'createYARNDirectories', - customCommand: 'CREATE_YARN_DIRECTORIES', - context: Em.I18n.t('services.service.actions.run.createYARNDirectories'), - label: Em.I18n.t('services.service.actions.run.createYARNDirectories'), - cssClass: 'icon-refresh' - }, REBALANCEHDFS: { action: 'rebalanceHdfsNodes', customCommand: 'REBALANCEHDFS', http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-web/app/views/main/service/item.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/views/main/service/item.js b/ambari-web/app/views/main/service/item.js index 02ac426..a007e17 100644 --- a/ambari-web/app/views/main/service/item.js +++ b/ambari-web/app/views/main/service/item.js @@ -40,8 +40,7 @@ App.MainServiceItemView = Em.View.extend({ 'HAWQMASTER': ['IMMEDIATE_STOP_HAWQ_SERVICE', 'RUN_HAWQ_CHECK', 'HAWQ_CLEAR_CACHE', 'REMOVE_HAWQ_STANDBY', 'RESYNC_HAWQ_STANDBY'], 'HAWQSEGMENT': ['IMMEDIATE_STOP_HAWQ_SEGMENT'], 'HAWQSTANDBY': ['ACTIVATE_HAWQ_STANDBY'], - 'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"], - 'NODEMANAGER' : ["CREATE_YARN_DIRECTORIES"] + 'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"] }, addActionMap: function() { @@ -142,16 +141,6 @@ App.MainServiceItemView = Em.View.extend({ } if (this.get('serviceName') === 'YARN') { options.push(actionMap.REFRESHQUEUES); - var nodeManagerComponent = App.StackServiceComponent.find().findProperty('componentName', 'NODEMANAGER'); - var isNodeManagerPresent = allSlaves.contains('NODEMANAGER'); - if (nodeManagerComponent && isNodeManagerPresent) { - var nodeManagerCustomCommands = nodeManagerComponent.get('customCommands'); - nodeManagerCustomCommands.forEach(function (command) { - if (actionMap[command]) { - options.push(actionMap[command]); - } - }); - } } options.push(actionMap.RESTART_ALL); allSlaves.concat(allMasters).filter(function (_component) { http://git-wip-us.apache.org/repos/asf/ambari/blob/da1701de/ambari-web/test/views/main/service/item_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/views/main/service/item_test.js b/ambari-web/test/views/main/service/item_test.js index ecfa7de..f318185 100644 --- a/ambari-web/test/views/main/service/item_test.js +++ b/ambari-web/test/views/main/service/item_test.js @@ -494,8 +494,6 @@ describe('App.MainServiceItemView', function () { return Em.Object.create({ customCommands: ["DECOMMISSION", "REBALANCEHDFS"] }); case 'RESOURCEMANAGER': return Em.Object.create({ customCommands: ["DECOMMISSION", "REFRESHQUEUES"] }); - case 'NODEMANAGER': - return Em.Object.create({ customCommands: ["REFRESHQUEUES", "CREATE_YARN_DIRECTORIES"] }); case 'HBASE_MASTER': return Em.Object.create({ customCommands: ["DECOMMISSION"] }); case 'KNOX_GATEWAY':
