Repository: ambari Updated Branches: refs/heads/branch-1.7.0 8560215cf -> e5c08c5b5 refs/heads/trunk 73b6d6b13 -> 482b7686c
AMBARI-8076. Install on a 5 node cluster fails with link creation for libsnappy. (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/482b7686 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/482b7686 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/482b7686 Branch: refs/heads/trunk Commit: 482b7686c1616cbd1d916b0732c32c5602f78fa1 Parents: 73b6d6b Author: Andrew Onishuk <[email protected]> Authored: Fri Oct 31 21:38:19 2014 +0200 Committer: Andrew Onishuk <[email protected]> Committed: Fri Oct 31 21:38:19 2014 +0200 ---------------------------------------------------------------------- .../1.3.2/hooks/before-START/scripts/params.py | 1 + .../scripts/shared_initialization.py | 3 +- .../2.0.6/hooks/before-START/scripts/params.py | 1 + .../scripts/shared_initialization.py | 3 +- .../hooks/before-START/test_before_start.py | 69 ++++++++++++++++-- .../hooks/before-START/test_before_start.py | 75 +++++++++++++++++--- 6 files changed, 136 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/482b7686/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py index fc66011..0788c50 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py @@ -33,6 +33,7 @@ user_group = config['configurations']['cluster-env']['user_group'] #hosts hostname = config["hostname"] +current_service = config['serviceName'] ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] rm_host = default("/clusterHostInfo/rm_host", []) slave_hosts = default("/clusterHostInfo/slave_hosts", []) http://git-wip-us.apache.org/repos/asf/ambari/blob/482b7686/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/shared_initialization.py index 81abf65..172abb1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/shared_initialization.py @@ -31,7 +31,8 @@ def setup_hadoop(): only_if="test -f /selinux/enforce" ) - install_snappy() + if params.current_service == "HDFS": + install_snappy() if params.has_namenode: http://git-wip-us.apache.org/repos/asf/ambari/blob/482b7686/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py index baf36cc..5bbd36f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py @@ -42,6 +42,7 @@ else: hadoop_bin = "/usr/lib/hadoop/sbin" hadoop_home = '/usr' +current_service = config['serviceName'] hadoop_conf_dir = "/etc/hadoop/conf" #security params security_enabled = config['configurations']['cluster-env']['security_enabled'] http://git-wip-us.apache.org/repos/asf/ambari/blob/482b7686/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py index f70eee8..6fda3f7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py @@ -31,7 +31,8 @@ def setup_hadoop(): only_if="test -f /selinux/enforce" ) - install_snappy() + if params.current_service == "HDFS": + install_snappy() #directories if params.has_namenode: http://git-wip-us.apache.org/repos/asf/ambari/blob/482b7686/ambari-server/src/test/python/stacks/1.3.2/hooks/before-START/test_before_start.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/1.3.2/hooks/before-START/test_before_start.py b/ambari-server/src/test/python/stacks/1.3.2/hooks/before-START/test_before_start.py index d569a95..d27ed5a 100644 --- a/ambari-server/src/test/python/stacks/1.3.2/hooks/before-START/test_before_start.py +++ b/ambari-server/src/test/python/stacks/1.3.2/hooks/before-START/test_before_start.py @@ -21,6 +21,7 @@ limitations under the License. from mock.mock import MagicMock, call, patch from resource_management import * from stacks.utils.RMFTestCase import * +import json @patch("os.path.exists", new = MagicMock(return_value=True)) @patch.object(Hook, "run_custom_hook") @@ -35,10 +36,6 @@ class TestHookBeforeStart(RMFTestCase): self.assertResourceCalled('Execute', '/bin/echo 0 > /selinux/enforce', only_if = 'test -f /selinux/enforce', ) - self.assertResourceCalled('Execute', 'mkdir -p /usr/lib/hadoop/lib/native/Linux-i386-32; ln -sf /usr/lib/libsnappy.so /usr/lib/hadoop/lib/native/Linux-i386-32/libsnappy.so', - ) - self.assertResourceCalled('Execute', 'mkdir -p /usr/lib/hadoop/lib/native/Linux-amd64-64; ln -sf /usr/lib64/libsnappy.so /usr/lib/hadoop/lib/native/Linux-amd64-64/libsnappy.so', - ) self.assertResourceCalled('Directory', '/var/log/hadoop', owner = 'root', group = 'hadoop', @@ -94,6 +91,62 @@ class TestHookBeforeStart(RMFTestCase): self.assertResourceCalled('Execute', '/bin/echo 0 > /selinux/enforce', only_if = 'test -f /selinux/enforce', ) + self.assertResourceCalled('Directory', '/var/log/hadoop', + owner = 'root', + group = 'hadoop', + mode = 0775, + recursive = True, + ) + self.assertResourceCalled('Directory', '/var/run/hadoop', + owner = 'root', + group = 'root', + recursive = True, + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/health_check', + content = Template('health_check.j2'), + owner = 'root', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/log4j.properties', + content = 'log4jproperties\nline2', + owner = 'hdfs', + group = 'hadoop', + mode = 0644, + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/hadoop-metrics2.properties', + content = Template('hadoop-metrics2.properties.j2'), + owner = 'hdfs', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/task-log4j.properties', + content = StaticFile('task-log4j.properties'), + mode = 0755, + ) + self.assertResourceCalled('Link', '/usr/lib/hadoop/lib/hadoop-tools.jar', + to = '/usr/lib/hadoop/hadoop-tools.jar', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/configuration.xsl', + owner = 'hdfs', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/masters', + owner = 'hdfs', + group = 'hadoop', + ) + self.assertNoMoreResources() + + def test_hook_default_hdfs(self, mockHook): + config_file = "stacks/1.3.2/configs/default.json" + with open(config_file, "r") as f: + default_json = json.load(f) + + default_json['serviceName']= 'HDFS' + self.executeScript("1.3.2/hooks/before-START/scripts/hook.py", + classname="BeforeStartHook", + command="hook", + config_dict=default_json + ) + self.assertResourceCalled('Execute', '/bin/echo 0 > /selinux/enforce', + only_if = 'test -f /selinux/enforce', + ) self.assertResourceCalled('Execute', 'mkdir -p /usr/lib/hadoop/lib/native/Linux-i386-32; ln -sf /usr/lib/libsnappy.so /usr/lib/hadoop/lib/native/Linux-i386-32/libsnappy.so', ) self.assertResourceCalled('Execute', 'mkdir -p /usr/lib/hadoop/lib/native/Linux-amd64-64; ln -sf /usr/lib64/libsnappy.so /usr/lib/hadoop/lib/native/Linux-amd64-64/libsnappy.so', @@ -109,12 +162,16 @@ class TestHookBeforeStart(RMFTestCase): group = 'root', recursive = True, ) + self.assertResourceCalled('File', '/etc/hadoop/conf/commons-logging.properties', + content = Template('commons-logging.properties.j2'), + owner = 'hdfs', + ) self.assertResourceCalled('File', '/etc/hadoop/conf/health_check', content = Template('health_check.j2'), - owner = 'root', + owner = 'hdfs', ) self.assertResourceCalled('File', '/etc/hadoop/conf/log4j.properties', - content = 'log4jproperties\nline2', + content = 'log4jproperties\nline2log4jproperties\nline2\nambari.jobhistory.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nambari.jobhistory.driver=org.postgresql.Driver\nambari.jobhistory.user=mapred\nambari.jobhistory.password=mapred\nambari.jobhistory.logger=${hadoop.root.logger}\n\nlog4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender\nlog4j.appender.JHA.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nlog4j.appender.JHA.driver=org.postgresql.Driver\nlog4j.appender.JHA.user=mapred\nlog4j.appender.JHA.password=mapred\n\nlog4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=DEBUG,JHA\nlog4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true\n\n', owner = 'hdfs', group = 'hadoop', mode = 0644, http://git-wip-us.apache.org/repos/asf/ambari/blob/482b7686/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py index 54b956e..4b7ffa5 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py +++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py @@ -21,6 +21,7 @@ limitations under the License. from mock.mock import MagicMock, call, patch from resource_management import * from stacks.utils.RMFTestCase import * +import json @patch("os.path.exists", new = MagicMock(return_value=True)) @patch.object(Hook, "run_custom_hook", new = MagicMock()) @@ -34,10 +35,6 @@ class TestHookBeforeStart(RMFTestCase): self.assertResourceCalled('Execute', '/bin/echo 0 > /selinux/enforce', only_if = 'test -f /selinux/enforce', ) - self.assertResourceCalled('Execute', 'mkdir -p /usr/lib/hadoop/lib/native/Linux-i386-32; ln -sf /usr/lib/libsnappy.so /usr/lib/hadoop/lib/native/Linux-i386-32/libsnappy.so', - ) - self.assertResourceCalled('Execute', 'mkdir -p /usr/lib/hadoop/lib/native/Linux-amd64-64; ln -sf /usr/lib64/libsnappy.so /usr/lib/hadoop/lib/native/Linux-amd64-64/libsnappy.so', - ) self.assertResourceCalled('Directory', '/var/log/hadoop', owner = 'root', group = 'hadoop', @@ -95,6 +92,68 @@ class TestHookBeforeStart(RMFTestCase): self.assertResourceCalled('Execute', '/bin/echo 0 > /selinux/enforce', only_if = 'test -f /selinux/enforce', ) + self.assertResourceCalled('Directory', '/var/log/hadoop', + owner = 'root', + group = 'hadoop', + mode = 0775, + recursive = True, + ) + self.assertResourceCalled('Directory', '/var/run/hadoop', + owner = 'root', + group = 'root', + recursive = True, + ) + self.assertResourceCalled('Directory', '/tmp/hadoop-hdfs', + owner = 'hdfs', + recursive = True, + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/commons-logging.properties', + content = Template('commons-logging.properties.j2'), + owner = 'root', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/health_check', + content = Template('health_check-v2.j2'), + owner = 'root', + ) + self.assertResourceCalled('File', + '/etc/hadoop/conf/log4j.properties', + mode=0644, + group='hadoop', + owner='hdfs', + content='log4jproperties\nline2log4jproperties\nline2' + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/hadoop-metrics2.properties', + content = Template('hadoop-metrics2.properties.j2'), + owner = 'hdfs', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/task-log4j.properties', + content = StaticFile('task-log4j.properties'), + mode = 0755, + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/configuration.xsl', + owner = 'hdfs', + group = 'hadoop', + ) + self.assertResourceCalled('File', '/etc/hadoop/conf/masters', + owner = 'hdfs', + group = 'hadoop', + ) + self.assertNoMoreResources() + + def test_hook_default_hdfs(self): + config_file = "stacks/2.0.6/configs/default.json" + with open(config_file, "r") as f: + default_json = json.load(f) + + default_json['serviceName']= 'HDFS' + self.executeScript("2.0.6/hooks/before-START/scripts/hook.py", + classname="BeforeStartHook", + command="hook", + config_dict=default_json + ) + self.assertResourceCalled('Execute', '/bin/echo 0 > /selinux/enforce', + only_if = 'test -f /selinux/enforce', + ) self.assertResourceCalled('Execute', 'mkdir -p /usr/lib/hadoop/lib/native/Linux-i386-32; ln -sf /usr/lib/libsnappy.so /usr/lib/hadoop/lib/native/Linux-i386-32/libsnappy.so', ) self.assertResourceCalled('Execute', 'mkdir -p /usr/lib/hadoop/lib/native/Linux-amd64-64; ln -sf /usr/lib64/libsnappy.so /usr/lib/hadoop/lib/native/Linux-amd64-64/libsnappy.so', @@ -116,11 +175,11 @@ class TestHookBeforeStart(RMFTestCase): ) self.assertResourceCalled('File', '/etc/hadoop/conf/commons-logging.properties', content = Template('commons-logging.properties.j2'), - owner = 'root', + owner = 'hdfs', ) self.assertResourceCalled('File', '/etc/hadoop/conf/health_check', content = Template('health_check-v2.j2'), - owner = 'root', + owner = 'hdfs', ) self.assertResourceCalled('File', '/etc/hadoop/conf/log4j.properties', @@ -128,7 +187,7 @@ class TestHookBeforeStart(RMFTestCase): group='hadoop', owner='hdfs', content='log4jproperties\nline2log4jproperties\nline2' - ) + ) self.assertResourceCalled('File', '/etc/hadoop/conf/hadoop-metrics2.properties', content = Template('hadoop-metrics2.properties.j2'), owner = 'hdfs', @@ -146,7 +205,7 @@ class TestHookBeforeStart(RMFTestCase): group = 'hadoop', ) self.assertNoMoreResources() - + def test_that_jce_is_required_in_secured_cluster(self): try: self.executeScript("2.0.6/hooks/before-START/scripts/hook.py",
