Repository: ambari Updated Branches: refs/heads/trunk 0276ecdea -> 643f51ee2
AMBARI-5485. Ambari should Manage JobHistory Heap Size (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/643f51ee Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/643f51ee Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/643f51ee Branch: refs/heads/trunk Commit: 643f51ee270d3a556e395c9acce8f5db46c31bf9 Parents: 0276ecd Author: Andrew Onishuk <[email protected]> Authored: Thu Apr 24 16:35:30 2014 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Thu Apr 24 16:35:30 2014 +0300 ---------------------------------------------------------------------- .../2.0.6/hooks/before-START/scripts/params.py | 3 +++ .../scripts/shared_initialization.py | 2 +- .../before-START/templates/mapred-env.sh.j2 | 27 ++++++++++++++++++++ .../YARN/configuration-mapred/global.xml | 6 ++++- .../hooks/before-START/test_before_start.py | 8 ++++++ .../controllers/main/service/info/configs.js | 2 +- .../app/controllers/wizard/step8_controller.js | 2 +- ambari-web/app/data/HDP2/global_properties.js | 14 ++++++++++ 8 files changed, 60 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/643f51ee/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py index 8b342f7..a8500c0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py @@ -105,6 +105,9 @@ if rca_enabled == True: else: rca_prefix = rca_disabled_prefix +#mapred-env.sh +jobhistory_heapsize = config['configurations']['global']['jobhistory_heapsize'] + #hadoop-env.sh java_home = config['hostLevelParams']['java_home'] http://git-wip-us.apache.org/repos/asf/ambari/blob/643f51ee/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py index e420f45..2a7ac77 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py @@ -60,7 +60,7 @@ def setup_hadoop(): else: tc_owner = params.hdfs_user - for file in ['hadoop-env.sh', 'commons-logging.properties']: + for file in ['hadoop-env.sh', 'commons-logging.properties', 'mapred-env.sh']: File(os.path.join(params.hadoop_conf_dir, file), owner=tc_owner, content=Template(file + ".j2") http://git-wip-us.apache.org/repos/asf/ambari/blob/643f51ee/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/mapred-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/mapred-env.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/mapred-env.sh.j2 new file mode 100644 index 0000000..7570fd9 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/mapred-env.sh.j2 @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# export JAVA_HOME=/home/y/libexec/jdk1.6.0/ + +export HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}} + +export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA + +#export HADOOP_JOB_HISTORYSERVER_OPTS= +#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default. +#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger. +#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default. +#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default +#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0. http://git-wip-us.apache.org/repos/asf/ambari/blob/643f51ee/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/global.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/global.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/global.xml index d02070c..e21a11e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/global.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/global.xml @@ -36,5 +36,9 @@ <value>mapred</value> <description>Mapreduce User</description> </property> - + <property> + <name>jobhistory_heapsize</name> + <value>900</value> + <description>Value for JobHistoryServer heap_size variable in hadoop-env.sh</description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/643f51ee/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py index c49bb53..5a15689 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py +++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py @@ -63,6 +63,10 @@ class TestHookBeforeStart(RMFTestCase): content = Template('commons-logging.properties.j2'), owner = 'hdfs', ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'hdfs', + ) self.assertResourceCalled('File', '/etc/hadoop/conf/health_check', content = Template('health_check-v2.j2'), owner = 'hdfs', @@ -145,6 +149,10 @@ class TestHookBeforeStart(RMFTestCase): content = Template('commons-logging.properties.j2'), owner = 'root', ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'root', + ) self.assertResourceCalled('File', '/etc/hadoop/conf/health_check', content = Template('health_check-v2.j2'), owner = 'root', http://git-wip-us.apache.org/repos/asf/ambari/blob/643f51ee/ambari-web/app/controllers/main/service/info/configs.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/service/info/configs.js b/ambari-web/app/controllers/main/service/info/configs.js index bd97b7c..b693666 100644 --- a/ambari-web/app/controllers/main/service/info/configs.js +++ b/ambari-web/app/controllers/main/service/info/configs.js @@ -1444,7 +1444,7 @@ App.MainServiceInfoConfigsController = Em.Controller.extend({ * @return {Object} */ createGlobalSiteObj: function (tagName, globalConfigs) { - var heapsizeException = ['hadoop_heapsize', 'yarn_heapsize', 'nodemanager_heapsize', 'resourcemanager_heapsize', 'apptimelineserver_heapsize']; + var heapsizeException = ['hadoop_heapsize', 'yarn_heapsize', 'nodemanager_heapsize', 'resourcemanager_heapsize', 'apptimelineserver_heapsize', 'jobhistory_heapsize']; var globalSiteProperties = {}; globalConfigs.forEach(function (_globalSiteObj) { // do not pass any globalConfigs whose name ends with _host or _hosts http://git-wip-us.apache.org/repos/asf/ambari/blob/643f51ee/ambari-web/app/controllers/wizard/step8_controller.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/wizard/step8_controller.js b/ambari-web/app/controllers/wizard/step8_controller.js index cdc9d48..bb79fc0 100644 --- a/ambari-web/app/controllers/wizard/step8_controller.js +++ b/ambari-web/app/controllers/wizard/step8_controller.js @@ -1510,7 +1510,7 @@ App.WizardStep8Controller = Em.Controller.extend({ } globalSiteObj.forEach(function (_globalSiteObj) { - var heapsizeException = ['hadoop_heapsize', 'yarn_heapsize', 'nodemanager_heapsize', 'resourcemanager_heapsize', 'apptimelineserver_heapsize']; + var heapsizeException = ['hadoop_heapsize', 'yarn_heapsize', 'nodemanager_heapsize', 'resourcemanager_heapsize', 'apptimelineserver_heapsize', 'jobhistory_heapsize']; // do not pass any globals whose name ends with _host or _hosts if (_globalSiteObj.isRequiredByAgent !== false) { // append "m" to JVM memory options except for hadoop_heapsize http://git-wip-us.apache.org/repos/asf/ambari/blob/643f51ee/ambari-web/app/data/HDP2/global_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/HDP2/global_properties.js b/ambari-web/app/data/HDP2/global_properties.js index cf5fbaf..b677416 100644 --- a/ambari-web/app/data/HDP2/global_properties.js +++ b/ambari-web/app/data/HDP2/global_properties.js @@ -276,6 +276,20 @@ module.exports = }, { "id": "puppet var", + "name": "jobhistory_heapsize", + "displayName": "History Server heap size", + "description": "History Server heap size", + "defaultValue": "900", + "unit": "MB", + "isOverridable": true, + "displayType": "int", + "isVisible": true, + "serviceName": "MAPREDUCE2", + "category": "HistoryServer", + "index": 1 + }, + { + "id": "puppet var", "name": "mapred_log_dir_prefix", "displayName": "Mapreduce Log Dir Prefix", "description": "",
