Repository: ambari Updated Branches: refs/heads/trunk 852d8af2f -> c4a9dc131
AMBARI-5611. Datanode and other components fail to start without YARN installed (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c4a9dc13 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c4a9dc13 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c4a9dc13 Branch: refs/heads/trunk Commit: c4a9dc131ec915ae2edaefb9cd7f032ac9de4b1a Parents: 852d8af Author: Andrew Onishuk <[email protected]> Authored: Tue Apr 29 16:40:32 2014 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Tue Apr 29 16:40:32 2014 +0300 ---------------------------------------------------------------------- .../2.0.6/hooks/before-START/scripts/params.py | 3 --- .../scripts/shared_initialization.py | 4 +-- .../before-START/templates/mapred-env.sh.j2 | 27 -------------------- .../services/YARN/package/scripts/params.py | 1 + .../2.0.6/services/YARN/package/scripts/yarn.py | 5 ++++ .../YARN/package/templates/mapred-env.sh.j2 | 27 ++++++++++++++++++++ .../stacks/2.0.6/YARN/test_historyserver.py | 10 +++++++- .../stacks/2.0.6/YARN/test_mapreduce2_client.py | 8 ++++++ .../stacks/2.0.6/YARN/test_nodemanager.py | 10 +++++++- .../stacks/2.0.6/YARN/test_resourcemanager.py | 10 +++++++- .../stacks/2.0.6/YARN/test_yarn_client.py | 12 +++++++++ .../hooks/before-START/test_before_start.py | 8 ------ .../stacks/2.1/YARN/test_apptimelineserver.py | 4 +++ 13 files changed, 86 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a9dc13/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py index a8500c0..8b342f7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py @@ -105,9 +105,6 @@ if rca_enabled == True: else: rca_prefix = rca_disabled_prefix -#mapred-env.sh -jobhistory_heapsize = config['configurations']['global']['jobhistory_heapsize'] - #hadoop-env.sh java_home = config['hostLevelParams']['java_home'] http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a9dc13/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py index 2a7ac77..f71b76c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py @@ -60,7 +60,7 @@ def setup_hadoop(): else: tc_owner = params.hdfs_user - for file in ['hadoop-env.sh', 'commons-logging.properties', 'mapred-env.sh']: + for file in ['hadoop-env.sh', 'commons-logging.properties']: File(os.path.join(params.hadoop_conf_dir, file), owner=tc_owner, content=Template(file + ".j2") @@ -194,4 +194,4 @@ def init_services(): ) - \ No newline at end of file + http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a9dc13/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/mapred-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/mapred-env.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/mapred-env.sh.j2 deleted file mode 100644 index 7570fd9..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/mapred-env.sh.j2 +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# export JAVA_HOME=/home/y/libexec/jdk1.6.0/ - -export HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}} - -export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA - -#export HADOOP_JOB_HISTORYSERVER_OPTS= -#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default. -#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger. -#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default. -#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default -#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0. http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a9dc13/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py index ca204f6..c87135a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py @@ -105,6 +105,7 @@ yarn_log_aggregation_enabled = config['configurations']['yarn-site']['yarn.log-a yarn_nm_app_log_dir = config['configurations']['yarn-site']['yarn.nodemanager.remote-app-log-dir'] mapreduce_jobhistory_intermediate_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.intermediate-done-dir'] mapreduce_jobhistory_done_dir = config['configurations']['mapred-site']['mapreduce.jobhistory.done-dir'] +jobhistory_heapsize = config['configurations']['global']['jobhistory_heapsize'] #for create_hdfs_directory hostname = config["hostname"] http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a9dc13/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py index 0ae7f52..cfb21e5 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py @@ -159,6 +159,11 @@ def yarn(name = None): tc_mode = None tc_owner = params.hdfs_user + File(format("{config_dir}/mapred-env.sh"), + owner=tc_owner, + content=Template('mapred-env.sh.j2') + ) + if params.security_enabled: File(os.path.join(params.hadoop_bin, "task-controller"), owner="root", http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a9dc13/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/mapred-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/mapred-env.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/mapred-env.sh.j2 new file mode 100644 index 0000000..7570fd9 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/templates/mapred-env.sh.j2 @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# export JAVA_HOME=/home/y/libexec/jdk1.6.0/ + +export HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}} + +export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA + +#export HADOOP_JOB_HISTORYSERVER_OPTS= +#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default. +#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger. +#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default. +#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default +#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0. http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a9dc13/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py index c831761..562f308 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py @@ -266,6 +266,10 @@ class TestHistoryServer(RMFTestCase): group = 'hadoop', mode = 0755, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'hdfs', + ) self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg', content = Template('taskcontroller.cfg.j2'), owner = 'hdfs', @@ -450,6 +454,10 @@ class TestHistoryServer(RMFTestCase): group = 'hadoop', mode = 0644, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'root', + ) self.assertResourceCalled('File', '/usr/lib/hadoop/sbin/task-controller', owner = 'root', group = 'hadoop', @@ -488,4 +496,4 @@ class TestHistoryServer(RMFTestCase): self.assertResourceCalled('File', '/etc/hadoop/conf/ssl-server.xml.example', owner = 'mapred', group = 'hadoop', - ) \ No newline at end of file + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a9dc13/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py index 850fcb6..30c9b87 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py @@ -125,6 +125,10 @@ class TestMapReduce2Client(RMFTestCase): group = 'hadoop', mode = 0755, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'hdfs', + ) self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg', content = Template('taskcontroller.cfg.j2'), owner = 'hdfs', @@ -256,6 +260,10 @@ class TestMapReduce2Client(RMFTestCase): group = 'hadoop', mode = 0644, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'root', + ) self.assertResourceCalled('File', '/usr/lib/hadoop/sbin/task-controller', owner = 'root', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a9dc13/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py index 51e6602..61aebcd 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py @@ -266,6 +266,10 @@ class TestNodeManager(RMFTestCase): group = 'hadoop', mode = 0755, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'hdfs', + ) self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg', content = Template('taskcontroller.cfg.j2'), owner = 'hdfs', @@ -450,6 +454,10 @@ class TestNodeManager(RMFTestCase): group = 'hadoop', mode = 0644, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'root', + ) self.assertResourceCalled('File', '/usr/lib/hadoop/sbin/task-controller', owner = 'root', group = 'hadoop', @@ -488,4 +496,4 @@ class TestNodeManager(RMFTestCase): self.assertResourceCalled('File', '/etc/hadoop/conf/ssl-server.xml.example', owner = 'mapred', group = 'hadoop', - ) \ No newline at end of file + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a9dc13/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py index ebf7ba9..0028046 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py @@ -210,6 +210,10 @@ class TestResourceManager(RMFTestCase): group = 'hadoop', mode = 0755, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'hdfs', + ) self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg', content = Template('taskcontroller.cfg.j2'), owner = 'hdfs', @@ -339,6 +343,10 @@ class TestResourceManager(RMFTestCase): group = 'hadoop', mode = 0644, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'root', + ) self.assertResourceCalled('File', '/usr/lib/hadoop/sbin/task-controller', owner = 'root', group = 'hadoop', @@ -377,4 +385,4 @@ class TestResourceManager(RMFTestCase): self.assertResourceCalled('File', '/etc/hadoop/conf/ssl-server.xml.example', owner = 'mapred', group = 'hadoop', - ) \ No newline at end of file + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a9dc13/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py index 33f6c7f..48545ae 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py @@ -126,6 +126,10 @@ class TestYarnClient(RMFTestCase): group = 'hadoop', mode = 0755, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'hdfs', + ) self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg', content = Template('taskcontroller.cfg.j2'), owner = 'hdfs', @@ -257,6 +261,10 @@ class TestYarnClient(RMFTestCase): group = 'hadoop', mode = 0644, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'root', + ) self.assertResourceCalled('File', '/usr/lib/hadoop/sbin/task-controller', owner = 'root', group = 'hadoop', @@ -397,6 +405,10 @@ class TestYarnClient(RMFTestCase): group = 'hadoop', mode = 0755, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'hdfs', + ) self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg', content = Template('taskcontroller.cfg.j2'), owner = 'hdfs', http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a9dc13/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py index 5a15689..c49bb53 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py +++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py @@ -63,10 +63,6 @@ class TestHookBeforeStart(RMFTestCase): content = Template('commons-logging.properties.j2'), owner = 'hdfs', ) - self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', - content = Template('mapred-env.sh.j2'), - owner = 'hdfs', - ) self.assertResourceCalled('File', '/etc/hadoop/conf/health_check', content = Template('health_check-v2.j2'), owner = 'hdfs', @@ -149,10 +145,6 @@ class TestHookBeforeStart(RMFTestCase): content = Template('commons-logging.properties.j2'), owner = 'root', ) - self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', - content = Template('mapred-env.sh.j2'), - owner = 'root', - ) self.assertResourceCalled('File', '/etc/hadoop/conf/health_check', content = Template('health_check-v2.j2'), owner = 'root', http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a9dc13/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py index 1848899..fc0559d 100644 --- a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py +++ b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py @@ -160,6 +160,10 @@ class TestAppTimelineServer(RMFTestCase): group = 'hadoop', mode = 0755, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh', + content = Template('mapred-env.sh.j2'), + owner = 'hdfs', + ) self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg', content = Template('taskcontroller.cfg.j2'), owner = 'hdfs',
