Repository: ambari Updated Branches: refs/heads/trunk 76785453b -> 8f67e1c80
AMBARI-16464 - Atlas fails to come up with solr as indexing search when the zookeeper quorum has more than one host Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8f67e1c8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8f67e1c8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8f67e1c8 Branch: refs/heads/trunk Commit: 8f67e1c80bfd11de34af763b98103183fbb0177a Parents: 7678545 Author: tbeerbower <[email protected]> Authored: Thu May 12 13:58:19 2016 -0400 Committer: tbeerbower <[email protected]> Committed: Thu May 12 13:58:19 2016 -0400 ---------------------------------------------------------------------- .../0.1.0.2.3/package/files/policy-store.txt | 26 ++++++++++++++++++++ .../package/files/users-credentials.properties | 7 +++--- .../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 7 ++++++ .../ATLAS/0.1.0.2.3/package/scripts/params.py | 21 ++++++++++++---- .../configuration/application-properties.xml | 7 ++++++ .../stacks/HDP/2.5/services/stack_advisor.py | 5 ++-- .../stacks/2.3/ATLAS/test_metadata_server.py | 6 +++++ .../stacks/2.5/ATLAS/test_atlas_server.py | 6 +++++ .../stacks/2.5/common/test_stack_advisor.py | 2 +- 9 files changed, 75 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8f67e1c8/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/policy-store.txt ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/policy-store.txt b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/policy-store.txt new file mode 100644 index 0000000..4b3b126 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/policy-store.txt @@ -0,0 +1,26 @@ +## +## Licensed to the Apache Software Foundation (ASF) under one +## or more contributor license agreements. See the NOTICE file +## distributed with this work for additional information +## regarding copyright ownership. The ASF licenses this file +## to you under the Apache License, Version 2.0 (the +## "License"); you may not use this file except in compliance +## with the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +##Policy Format +##r-READ, w-WRITE, u-UPDATE, d-DELETE +##Policy_Name;;User_Name1:Operations_Allowed,User_Name2:Operations_Allowed;;Group_Name1:Operations_Allowed,Group_Name2:Operations_Allowed;;Resource_Type1:Resource_Name,Resource_Type2:Resource_Name +## +adminPolicy;;admin:rwud;;ROLE_ADMIN:rwud;;type:*,entity:*,operation:* +typeReadPolicy;;nixon:rw;;;;type:*,entity:* +classReadPolicy;;saqeeb:r;;;;type:*,entity:* +dataScientistPolicy;;;;DATA_SCIENTIST:r;;type:*,entity:* +dataStewardPolicy;;;;DATA_STEWARD:rwu;;type:*,entity:* http://git-wip-us.apache.org/repos/asf/ambari/blob/8f67e1c8/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties index 4d264a0..33b326f 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/files/users-credentials.properties @@ -16,6 +16,7 @@ # limitations under the License. # -#username=password -admin=admin -user=user123 \ No newline at end of file +#username=group::sha256-password +admin=ADMIN::8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918 +michael=DATA_SCIENTIST::95bfb24de17d285d734b9eaa9109bfe922adc85f20d2e5e66a78bddb4a4ebddb +paul=DATA_STEWARD::e7c0dcf5f8a93e93791e9bac1ae454a691c1d2a902fc4256d489e96c1b9ac68c http://git-wip-us.apache.org/repos/asf/ambari/blob/8f67e1c8/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py index 8e2fcc2..3138161 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py @@ -114,6 +114,13 @@ def metadata(): content=StaticFile('users-credentials.properties') ) + File(format("{conf_dir}/policy-store.txt"), + mode=0644, + owner=params.metadata_user, + group=params.user_group, + content=StaticFile('policy-store.txt') + ) + if params.atlas_has_embedded_hbase: # hbase-site for embedded hbase used by Atlas XmlConfig( "hbase-site.xml", http://git-wip-us.apache.org/repos/asf/ambari/blob/8f67e1c8/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index 51fa039..f9422ee 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -62,6 +62,7 @@ conf_dir = status_params.conf_dir # "/etc/metadata/conf" conf_file = status_params.conf_file atlas_login_credentials_file = os.path.join(conf_dir, "users-credentials.properties") +atlas_policy_store_file = os.path.join(conf_dir, "policy-store.txt") atlas_hbase_conf_dir = os.path.join(metadata_home, "hbase", "conf") atlas_hbase_log_dir = os.path.join(metadata_home, "hbase", "logs") @@ -161,13 +162,23 @@ for host in atlas_hosts: id += 1 first_id = False +zookeeper_hosts = config['clusterHostInfo']['zookeeper_hosts'] zookeeper_port = default('/configurations/zoo.cfg/clientPort', None) -# get comma separated list of zookeeper hosts from clusterHostInfo +logsearch_solr_znode = default("/configurations/logsearch-solr-env/logsearch_solr_znode", None) + +# get comma separated lists of zookeeper hosts from clusterHostInfo index = 0 zookeeper_quorum = "" -for host in config['clusterHostInfo']['zookeeper_hosts']: - zookeeper_quorum += host + ":" + str(zookeeper_port) +solr_zookeeper_url = "" + +for host in zookeeper_hosts: + zookeeper_host = host + ":" + str(zookeeper_port) + + if logsearch_solr_znode is not None: + solr_zookeeper_url += zookeeper_host + logsearch_solr_znode + + zookeeper_quorum += zookeeper_host index += 1 - if index < len(config['clusterHostInfo']['zookeeper_hosts']): + if index < len(zookeeper_hosts): zookeeper_quorum += "," - + solr_zookeeper_url += "," \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/8f67e1c8/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml index 78600ee..03f825b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml @@ -65,6 +65,8 @@ <property> <name>atlas.graph.index.search.solr.zookeeper-url</name> <description>The ZooKeeper quorum setup for Solr as comma separated value.</description> + </property> + <property> <name>atlas.login.method</name> <value>file</value> <description>Atlas login method (e.g. file).</description> @@ -74,4 +76,9 @@ <value>{{atlas_login_credentials_file}}</value> <description>File path for file based login.</description> </property> + <property> + <name>atlas.auth.policy.file</name> + <value>{{atlas_policy_store_file}}</value> + <description>Path for the Atlas policy file.</description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/8f67e1c8/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 5ae3664..b954c5f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -134,8 +134,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): include_logsearch = "LOGSEARCH" in servicesList if include_logsearch and "logsearch-solr-env" in services["configurations"]: - logsearch_solr_znode = services["configurations"]["logsearch-solr-env"]["properties"]['logsearch_solr_znode'] - putAtlasApplicationProperty('atlas.graph.index.search.solr.zookeeper-url', '{{zookeeper_quorum}}' + logsearch_solr_znode) + putAtlasApplicationProperty('atlas.graph.index.search.solr.zookeeper-url', '{{solr_zookeeper_url}}') def recommendHIVEConfigurations(self, configurations, clusterData, services, hosts): super(HDP25StackAdvisor, self).recommendHIVEConfigurations(configurations, clusterData, services, hosts) @@ -1098,4 +1097,4 @@ class HDP25StackAdvisor(HDP24StackAdvisor): hosts_for_component = [] if serviceName in servicesList: hosts_for_component = [component["hostnames"] for component in componentsList if component["component_name"] == componentName][0] - return hosts_for_component \ No newline at end of file + return hosts_for_component http://git-wip-us.apache.org/repos/asf/ambari/blob/8f67e1c8/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py index 8ae2c11..80a345e 100644 --- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py +++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py @@ -116,6 +116,12 @@ class TestMetadataServer(RMFTestCase): group='hadoop', mode=0644, ) + self.assertResourceCalled('File', '/etc/atlas/conf/policy-store.txt', + content=StaticFile('policy-store.txt'), + owner='atlas', + group='hadoop', + mode=0644, + ) self.assertResourceCalled('XmlConfig', 'hbase-site.xml', owner = 'atlas', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/8f67e1c8/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py index 422b1c9..daa40d6 100644 --- a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py +++ b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py @@ -114,6 +114,12 @@ class TestAtlasServer(RMFTestCase): group='hadoop', mode=0644, ) + self.assertResourceCalled('File', '/etc/atlas/conf/policy-store.txt', + content=StaticFile('policy-store.txt'), + owner='atlas', + group='hadoop', + mode=0644, + ) self.assertResourceCalled('XmlConfig', 'hbase-site.xml', owner = 'atlas', group = 'hadoop', http://git-wip-us.apache.org/repos/asf/ambari/blob/8f67e1c8/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index fc4ebc6..cf82a9c 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -3471,7 +3471,7 @@ class TestHDP25StackAdvisor(TestCase): expected = { 'application-properties': { 'properties': { - 'atlas.graph.index.search.solr.zookeeper-url': '{{zookeeper_quorum}}/logsearch', + 'atlas.graph.index.search.solr.zookeeper-url': '{{solr_zookeeper_url}}', } }, "logsearch-solr-env": {
