This is an automated email from the ASF dual-hosted git repository. dsen pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push: new 6bcefac [AMBARI-23751] Ambari ViewFS: Add ability to add XML inclusion node to a configuration file (dsen) (#1168) 6bcefac is described below commit 6bcefac7720077204057f97bd5571db7529292da Author: Dmitry Sen <d...@apache.org> AuthorDate: Fri May 4 14:04:17 2018 +0300 [AMBARI-23751] Ambari ViewFS: Add ability to add XML inclusion node to a configuration file (dsen) (#1168) * [AMBARI-23751] Ambari ViewFS: Add ability to add XML inclusion node to a configuration file (dsen) * [AMBARI-23751] Ambari ViewFS: Add ability to add XML inclusion node to a configuration file (dsen) - UT fix and hooks change --- .../resource_management/TestXmlConfigResource.py | 36 +++++++++++++++++++--- .../libraries/providers/xml_config.py | 10 ++++-- .../libraries/resources/xml_config.py | 1 + .../stack-hooks/after-INSTALL/scripts/params.py | 10 ++++++ .../after-INSTALL/scripts/shared_initialization.py | 11 ++++++- .../hooks/after-INSTALL/test_after_install.py | 12 +++++--- 6 files changed, 68 insertions(+), 12 deletions(-) diff --git a/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py b/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py index 09eafc9..9b4c1d5 100644 --- a/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py +++ b/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py @@ -68,7 +68,7 @@ class TestXmlConfigResource(TestCase): configuration_attributes={} ) - create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration>\n \n </configuration>', encoding='UTF-8') + create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n </configuration>', encoding='UTF-8') @patch("resource_management.core.providers.system._ensure_metadata") @@ -97,8 +97,36 @@ class TestXmlConfigResource(TestCase): configuration_attributes={'attr': {'property1': 'attr_value'}} ) - create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration>\n \n <property>\n <name>property1</name>\n <value>value1</value>\n <attr>attr_value</attr>\n </property>\n \n </configuration>', encoding='UTF-8') + create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name>property1</name>\n <value>value1</value>\n <attr>attr_value</attr>\n </property>\n \n </configuration>', encoding='UTF-8') + @patch("resource_management.core.providers.system._ensure_metadata") + @patch("resource_management.core.sudo.create_file") + @patch("resource_management.core.sudo.path_exists") + @patch("resource_management.core.sudo.path_isdir") + @patch.object(time, "asctime") + def test_action_create_simple_xml_config_with_inclusion(self, + time_asctime_mock, + os_path_isdir_mock, + os_path_exists_mock, + create_file_mock, + ensure_mock): + """ + Tests if 'create' action - creates new non existent xml file and write proper data + where configurations={"Some conf":"Some value"} + """ + os_path_isdir_mock.side_effect = [False, True] + os_path_exists_mock.return_value = False + time_asctime_mock.return_value = 'Wed 2014-02' + + with Environment('/') as env: + XmlConfig('file.xml', + conf_dir='/dir/conf', + configurations={'property1': 'value1'}, + configuration_attributes={'attr': {'property1': 'attr_value'}}, + xml_include_file="/dif/conf/include_file.xml" + ) + + create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name>property1</name>\n <value>value1</value>\n <attr>attr_value</attr>\n </property>\n \n <xi:include href="/dif/conf/include_file.xml"/>\n \n </configuration>', encoding='UTF-8') @patch("resource_management.core.providers.system._ensure_metadata") @patch("resource_management.core.sudo.create_file") @@ -150,7 +178,7 @@ class TestXmlConfigResource(TestCase): } }) - create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration>\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>prop.1</name>\n <value>'.'yyyy-MM-dd-HH</value>\n <attr1>x</attr1>\n </property>\n \n <property>\n <name>prop.2</name>\n <value>INFO, openjpa</value>\n </property>\n \n <property>\n <name>prop.3</name>\n <value>%d{ISO8601} %5p %c{1}:%L [...] + create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>prop.1</name>\n <value>'.'yyyy-MM-dd-HH</value>\n <attr1>x</attr1>\n </property>\n \n <property>\n <name>prop.2</name>\n <value>INFO, openjpa</value>\n </property>\n \n <property>\n <name>prop.3</n [...] @patch("resource_management.core.providers.system._ensure_metadata") @patch("resource_management.core.sudo.create_file") @@ -183,7 +211,7 @@ class TestXmlConfigResource(TestCase): configuration_attributes={} ) - create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration>\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>first</name>\n <value>should be first</value>\n </property>\n \n <property>\n <name>second</name>\n <value>should be second</value>\n </property>\n \n <property>\n <name>third</name>\n <value>should be third</value>\n </property>\n \n <prop [...] + create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>first</name>\n <value>should be first</value>\n </property>\n \n <property>\n <name>second</name>\n <value>should be second</value>\n </property>\n \n <property>\n <name>third</name>\n <value>should be thi [...] @patch("resource_management.libraries.providers.xml_config.File") @patch("resource_management.core.sudo.path_exists") diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py index eebd0dd..bf196b5 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py @@ -38,7 +38,7 @@ class XmlConfigProvider(Provider): configuration_attrs = {} if is_empty(self.resource.configuration_attributes) else self.resource.configuration_attributes # |e - for html-like escaping of <,>,'," - config_content = InlineTemplate(''' <configuration> + config_content = InlineTemplate(''' <configuration xmlns:xi="http://www.w3.org/2001/XInclude"> {% for key, value in configurations_dict|dictsort %} <property> <name>{{ key|e }}</name> @@ -54,8 +54,12 @@ class XmlConfigProvider(Provider): {%- endif %} </property> {% endfor %} - </configuration>''', extra_imports=[time, resource_management, resource_management.core, resource_management.core.source], configurations_dict=self.resource.configurations, - configuration_attrs=configuration_attrs) + {%- if not xml_include_file is none %} + <xi:include href="{{xml_include_file|e}}"/> + {% endif %} + </configuration>''', extra_imports=[time, resource_management, resource_management.core, resource_management.core.source], + configurations_dict=self.resource.configurations, + configuration_attrs=configuration_attrs, xml_include_file=self.resource.xml_include_file) xml_config_dest_file_path = os.path.join(xml_config_provider_config_dir, filename) Logger.info("Generating config: {0}".format(xml_config_dest_file_path)) diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py b/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py index 882d6d5..c46f35c 100644 --- a/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py +++ b/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py @@ -34,6 +34,7 @@ class XmlConfig(Resource): mode = ResourceArgument() owner = ResourceArgument() group = ResourceArgument() + xml_include_file = ResourceArgument() encoding = ResourceArgument(default="UTF-8") diff --git a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py index 91c75dd..e66d6ac 100644 --- a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py @@ -102,6 +102,16 @@ has_namenode = not len(namenode_host) == 0 if has_namenode or dfs_type == 'HCFS': hadoop_conf_dir = conf_select.get_hadoop_conf_dir() + mount_table_xml_inclusion_file_full_path = None + mount_table_content = None + if 'mount-table' in config['configurations']: + xml_inclusion_file_name = 'mount-table.xml' + mount_table = config['configurations']['mount-table'] + + if 'content' in mount_table and mount_table['content'].strip(): + mount_table_xml_inclusion_file_full_path = os.path.join(hadoop_conf_dir, xml_inclusion_file_name) + mount_table_content = mount_table['content'] + link_configs_lock_file = get_config_lock_file() stack_select_lock_file = os.path.join(tmp_dir, "stack_select_lock_file") diff --git a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/shared_initialization.py index 22543bc..d82f565 100644 --- a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/shared_initialization.py @@ -85,7 +85,16 @@ def setup_config(): configuration_attributes=params.config['configurationAttributes']['core-site'], owner=params.hdfs_user, group=params.user_group, - only_if=format("ls {hadoop_conf_dir}")) + only_if=format("ls {hadoop_conf_dir}"), + xml_include_file=params.mount_table_xml_inclusion_file_full_path + ) + + if params.mount_table_content: + File(os.path.join(params.hadoop_conf_dir, params.xml_inclusion_file_name), + owner=params.hdfs_user, + group=params.user_group, + content=params.mount_table_content + ) Directory(params.logsearch_logfeeder_conf, mode=0755, diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py index 0791784..c03a10f 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py +++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py @@ -57,7 +57,8 @@ class TestHookAfterInstall(RMFTestCase): conf_dir = '/etc/hadoop/conf', configurations = self.getConfig()['configurations']['core-site'], configuration_attributes = self.getConfig()['configurationAttributes']['core-site'], - only_if="ls /etc/hadoop/conf") + only_if="ls /etc/hadoop/conf", + xml_include_file=None) self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder/conf', mode = 0755, @@ -104,7 +105,8 @@ class TestHookAfterInstall(RMFTestCase): conf_dir = "/usr/hdp/2.3.0.0-1234/hadoop/conf", configurations = self.getConfig()['configurations']['core-site'], configuration_attributes = self.getConfig()['configurationAttributes']['core-site'], - only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf") + only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf", + xml_include_file=None) self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder/conf', @@ -172,7 +174,8 @@ class TestHookAfterInstall(RMFTestCase): conf_dir = "/usr/hdp/2.3.0.0-1234/hadoop/conf", configurations = self.getConfig()['configurations']['core-site'], configuration_attributes = self.getConfig()['configurationAttributes']['core-site'], - only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf") + only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf", + xml_include_file=None) self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder/conf', @@ -275,7 +278,8 @@ class TestHookAfterInstall(RMFTestCase): conf_dir = "/usr/hdp/2.3.0.0-1234/hadoop/conf", configurations = self.getConfig()['configurations']['core-site'], configuration_attributes = self.getConfig()['configurationAttributes']['core-site'], - only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf") + only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf", + xml_include_file=None) self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-logfeeder/conf', -- To stop receiving notification emails like this one, please contact d...@apache.org.