Repository: ambari Updated Branches: refs/heads/trunk 1734b5c7b -> 5db6f1d21
AMBARI-15981 - Atlas Integration : Make default Atlas configuration for embedded Hbase Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5db6f1d2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5db6f1d2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5db6f1d2 Branch: refs/heads/trunk Commit: 5db6f1d21f05b61d9da1240da693350021984001 Parents: 1734b5c Author: tbeerbower <[email protected]> Authored: Fri Apr 22 07:20:06 2016 -0400 Committer: tbeerbower <[email protected]> Committed: Fri Apr 22 07:20:20 2016 -0400 ---------------------------------------------------------------------- .../configuration/application-properties.xml | 21 ++++++- .../ATLAS/0.1.0.2.3/configuration/atlas-env.xml | 2 +- .../configuration/atlas-hbase-site.xml | 58 ++++++++++++++++++++ .../ATLAS/0.1.0.2.3/metainfo.xml | 15 ++++- .../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 27 ++++++++- .../ATLAS/0.1.0.2.3/package/scripts/params.py | 5 ++ .../services/ATLAS/configuration/atlas-env.xml | 2 +- .../stacks/HDP/2.5/services/ATLAS/metainfo.xml | 4 +- .../stacks/2.3/ATLAS/test_metadata_server.py | 21 +++++++ .../test/python/stacks/2.3/configs/default.json | 4 ++ .../test/python/stacks/2.3/configs/secure.json | 4 ++ .../stacks/2.5/ATLAS/test_atlas_server.py | 21 +++++++ .../test/python/stacks/2.5/configs/default.json | 4 ++ 13 files changed, 178 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml index 95141ca..f3ea36f 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml @@ -21,12 +21,12 @@ <configuration supports_final="false"> <property> <name>atlas.graph.storage.backend</name> - <value>berkeleyje</value> + <value>hbase</value> <description></description> </property> <property> - <name>atlas.graph.storage.directory</name> - <value>/var/lib/atlas/data/berkeley</value> + <name>atlas.graph.storage.hostname</name> + <value>localhost:{{atlas_hbase_zk_port}}</value> <description></description> </property> <property> @@ -210,4 +210,19 @@ <value>1000</value> <description></description> </property> + <property> + <name>atlas.audit.hbase.tablename</name> + <value>ATLAS_ENTITY_AUDIT_EVENTS</value> + <description></description> + </property> + <property> + <name>atlas.audit.zookeeper.session.timeout.ms</name> + <value>1000</value> + <description></description> + </property> + <property> + <name>atlas.audit.hbase.zookeeper.quorum</name> + <value>localhost:{{atlas_hbase_zk_port}}</value> + <description></description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml index d64d48c..90eabef 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml @@ -102,7 +102,7 @@ export METADATA_DATA_DIR={{data_dir}} export METADATA_PID_DIR={{pid_dir}} # hbase conf dir -export HBASE_CONF_DIR=/etc/hbase/conf +export HBASE_CONF_DIR={{atlas_hbase_conf_dir}} # Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir. export METADATA_EXPANDED_WEBAPP_DIR={{expanded_war_dir}} http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml new file mode 100644 index 0000000..3c4826d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml @@ -0,0 +1,58 @@ +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="true"> + <property> + <name>hbase.rootdir</name> + <value>file://{{metadata_home}}/data/root</value> + </property> + <property> + <name>hbase.zookeeper.property.dataDir</name> + <value>{{metadata_home}}/data/zookeeper-data</value> + </property> + <property> + <name>hbase.master.info.port</name> + <value>61510</value> + </property> + <property> + <name>hbase.regionserver.info.port</name> + <value>61530</value> + </property> + <property> + <name>hbase.master.port</name> + <value>61500</value> + </property> + <property> + <name>hbase.regionserver.port</name> + <value>61520</value> + </property> + <property> + <name>hbase.zookeeper.property.clientPort</name> + <value>61581</value> + </property> + <property> + <name>hbase.zookeeper.property.peerport</name> + <value>61588</value> + </property> + <property> + <name>hbase.zookeeper.property.leaderport</name> + <value>61589</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml index 133b264..5d7766b 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml @@ -46,7 +46,7 @@ </logs> <configFile> <type>properties</type> - <fileName>application.properties</fileName> + <fileName>application.properties.xml</fileName> <dictionaryName>application-properties</dictionaryName> </configFile> <configFile> @@ -59,6 +59,11 @@ <fileName>atlas-log4j.xml</fileName> <dictionaryName>atlas-log4j</dictionaryName> </configFile> + <configFile> + <type>xml</type> + <fileName>atlas-hbase-site.xml</fileName> + <dictionaryName>atlas-hbase-site</dictionaryName> + </configFile> </component> <component> @@ -75,7 +80,7 @@ </commandScript> <configFile> <type>properties</type> - <fileName>application.properties</fileName> + <fileName>application.properties.xml</fileName> <dictionaryName>application-properties</dictionaryName> </configFile> <configFile> @@ -88,6 +93,11 @@ <fileName>atlas-log4j.xml</fileName> <dictionaryName>atlas-log4j</dictionaryName> </configFile> + <configFile> + <type>xml</type> + <fileName>atlas-hbase-site.xml</fileName> + <dictionaryName>atlas-hbase-site</dictionaryName> + </configFile> </component> </components> @@ -116,6 +126,7 @@ <config-type>application-properties</config-type> <config-type>atlas-env</config-type> <config-type>atlas-log4j</config-type> + <config-type>atlas-hbase-site</config-type> </configuration-dependencies> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py index 041e04e..43c1ea6 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py @@ -18,7 +18,7 @@ limitations under the License. """ from resource_management import Directory, Fail, Logger, File, \ - InlineTemplate, PropertiesFile, StaticFile + InlineTemplate, PropertiesFile, StaticFile, XmlConfig from resource_management.libraries.functions import format from resource_management.libraries.resources.template_config import TemplateConfig @@ -50,6 +50,22 @@ def metadata(): create_parents = True ) + Directory(params.atlas_hbase_log_dir, + mode=0755, + cd_access='a', + owner=params.metadata_user, + group=params.user_group, + create_parents = True + ) + + Directory(params.atlas_hbase_data_dir, + mode=0755, + cd_access='a', + owner=params.metadata_user, + group=params.user_group, + create_parents = True + ) + Directory(params.data_dir, mode=0644, cd_access='a', @@ -91,6 +107,15 @@ def metadata(): content=InlineTemplate(params.metadata_log4j_content) ) + # hbase-site for embedded hbase used by Atlas + XmlConfig( "hbase-site.xml", + conf_dir = params.atlas_hbase_conf_dir, + configurations = params.config['configurations']['atlas-hbase-site'], + configuration_attributes=params.config['configuration_attributes']['atlas-hbase-site'], + owner = params.metadata_user, + group = params.user_group + ) + if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user) http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index 04f6bf2..4191212 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -61,6 +61,11 @@ log_dir = config['configurations']['atlas-env']['metadata_log_dir'] conf_dir = status_params.conf_dir # "/etc/metadata/conf" conf_file = status_params.conf_file +atlas_hbase_conf_dir = os.path.join(metadata_home, "hbase", "conf") +atlas_hbase_log_dir = os.path.join(metadata_home, "hbase", "logs") +atlas_hbase_data_dir = os.path.join(metadata_home, "data") +atlas_hbase_zk_port = default("/configurations/atlas-hbase-site/hbase.zookeeper.property.clientPort", None) + # service locations hadoop_conf_dir = os.path.join(os.environ["HADOOP_HOME"], "conf") if 'HADOOP_HOME' in os.environ else '/etc/hadoop/conf' http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml index 42503b5..a2ea7e5 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml @@ -102,7 +102,7 @@ export ATLAS_PID_DIR={{pid_dir}} # hbase conf dir - export HBASE_CONF_DIR=/etc/hbase/conf + export HBASE_CONF_DIR={{atlas_hbase_conf_dir}} # Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir. export ATLAS_EXPANDED_WEBAPP_DIR={{expanded_war_dir}} http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml index c787b40..92f198a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml @@ -27,7 +27,7 @@ <cardinality>1+</cardinality> <configFile> <type>properties</type> - <fileName>atlas-application.properties</fileName> + <fileName>application.properties.xml</fileName> <dictionaryName>application-properties</dictionaryName> </configFile> <logs> @@ -42,7 +42,7 @@ <name>ATLAS_CLIENT</name> <configFile> <type>properties</type> - <fileName>atlas-application.properties</fileName> + <fileName>application.properties.xml</fileName> <dictionaryName>application-properties</dictionaryName> </configFile> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py index b1db109..cf18f92 100644 --- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py +++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py @@ -52,6 +52,20 @@ class TestMetadataServer(RMFTestCase): cd_access='a', mode=0755 ) + self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/hbase/logs', + owner='atlas', + group='hadoop', + create_parents = True, + cd_access='a', + mode=0755 + ) + self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/data', + owner='atlas', + group='hadoop', + create_parents = True, + cd_access='a', + mode=0755 + ) self.assertResourceCalled('Directory', '/var/lib/atlas/data', owner='atlas', group='hadoop', @@ -96,6 +110,13 @@ class TestMetadataServer(RMFTestCase): group='hadoop', mode=0644, ) + self.assertResourceCalled('XmlConfig', 'hbase-site.xml', + owner = 'atlas', + group = 'hadoop', + conf_dir = '/usr/hdp/current/atlas-server/hbase/conf', + configurations = self.getConfig()['configurations']['atlas-hbase-site'], + configuration_attributes = self.getConfig()['configuration_attributes']['atlas-hbase-site'] + ) def test_configure_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py", http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/test/python/stacks/2.3/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json index f87a4b8..d423ce5 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json @@ -217,6 +217,9 @@ "atlas_log_level": "debug", "audit_log_level": "OFF" }, + "atlas-hbase-site": { + "hbase.master.info.port": "61510" + }, "ranger-hbase-plugin-properties": { "ranger-hbase-plugin-enabled":"yes" }, @@ -261,6 +264,7 @@ "kafka-env": {}, "kafka-log4j": {}, "kafka-broker": {}, + "atlas-hbase-site": {}, "metadata-env": {} }, "configurationTags": { http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/test/python/stacks/2.3/configs/secure.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/secure.json b/ambari-server/src/test/python/stacks/2.3/configs/secure.json index 524484f..89ab6f0 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/secure.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/secure.json @@ -209,6 +209,9 @@ "atlas_log_level": "debug", "audit_log_level": "OFF" }, + "atlas-hbase-site": { + "hbase.master.info.port": "61510" + }, "ranger-hbase-plugin-properties": { "ranger-hbase-plugin-enabled":"yes" }, @@ -252,6 +255,7 @@ "kafka-env": {}, "kafka-log4j": {}, "kafka-broker": {}, + "atlas-hbase-site": {}, "metadata-env": {} }, "configurationTags": { http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py index a882189..83d4c5b 100644 --- a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py +++ b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py @@ -49,6 +49,20 @@ class TestAtlasServer(RMFTestCase): cd_access='a', mode=0755 ) + self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/hbase/logs', + owner='atlas', + group='hadoop', + create_parents = True, + cd_access='a', + mode=0755 + ) + self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/data', + owner='atlas', + group='hadoop', + create_parents = True, + cd_access='a', + mode=0755 + ) self.assertResourceCalled('Directory', '/var/lib/atlas/data', owner='atlas', group='hadoop', @@ -94,6 +108,13 @@ class TestAtlasServer(RMFTestCase): group='hadoop', mode=0644, ) + self.assertResourceCalled('XmlConfig', 'hbase-site.xml', + owner = 'atlas', + group = 'hadoop', + conf_dir = '/usr/hdp/current/atlas-server/hbase/conf', + configurations = self.getConfig()['configurations']['atlas-hbase-site'], + configuration_attributes = self.getConfig()['configuration_attributes']['atlas-hbase-site'] + ) def test_configure_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py", http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/test/python/stacks/2.5/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/configs/default.json b/ambari-server/src/test/python/stacks/2.5/configs/default.json index ddf4c9a..5188730 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/default.json @@ -217,6 +217,9 @@ "atlas_log_level": "debug", "audit_log_level": "OFF" }, + "atlas-hbase-site": { + "hbase.master.info.port": "61510" + }, "ranger-hbase-plugin-properties": { "ranger-hbase-plugin-enabled":"yes" }, @@ -288,6 +291,7 @@ "kafka-log4j": {}, "kafka-broker": {}, "metadata-env": {}, + "atlas-hbase-site": {}, "tagsync-application-properties": {}, "ranger-tagsync-site": {} },
