Repository: ambari Updated Branches: refs/heads/trunk 025b3446f -> 4104f2f9d
AMBARI-16089 - Atlas Integration : set atlas.cluster.name in hive-site Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4104f2f9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4104f2f9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4104f2f9 Branch: refs/heads/trunk Commit: 4104f2f9d61dbc3d01ea8dfcbfe9faa6ece2b03c Parents: 025b344 Author: tbeerbower <[email protected]> Authored: Wed Apr 27 13:57:34 2016 -0400 Committer: tbeerbower <[email protected]> Committed: Wed Apr 27 13:57:47 2016 -0400 ---------------------------------------------------------------------- .../ATLAS/0.1.0.2.3/package/scripts/params.py | 2 + .../HIVE/0.12.0.2.0/configuration/hive-site.xml | 2 +- .../0.12.0.2.0/package/scripts/params_linux.py | 2 + .../stacks/HDP/2.3/services/stack_advisor.py | 2 - .../configuration/application-properties.xml | 8 + .../stacks/2.3/common/test_stack_advisor.py | 268 +++++++++++++++++++ 6 files changed, 281 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4104f2f9/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index c86dbf9..defd6f0 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -61,6 +61,8 @@ log_dir = config['configurations']['atlas-env']['metadata_log_dir'] conf_dir = status_params.conf_dir # "/etc/metadata/conf" conf_file = status_params.conf_file +atlas_login_credentials_file = os.path.join(conf_dir, "users-credentials.properties") + atlas_hbase_conf_dir = os.path.join(metadata_home, "hbase", "conf") atlas_hbase_log_dir = os.path.join(metadata_home, "hbase", "logs") atlas_hbase_data_dir = os.path.join(metadata_home, "data") http://git-wip-us.apache.org/repos/asf/ambari/blob/4104f2f9/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml index e56c3e9..2a2bb23 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml @@ -442,7 +442,7 @@ limitations under the License. <property> <name>atlas.cluster.name</name> <property-type>DONT_ADD_ON_UPGRADE</property-type> - <value>primary</value> + <value>{{cluster_name}}</value> <depends-on> <property> <type>application-properties</type> http://git-wip-us.apache.org/repos/asf/ambari/blob/4104f2f9/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 3d6c096..d0c3b3a 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -52,6 +52,8 @@ stack_name_uppercase = stack_name.upper() agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability'] agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) +cluster_name = config['clusterName'] + # node hostname hostname = config["hostname"] http://git-wip-us.apache.org/repos/asf/ambari/blob/4104f2f9/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index f2e9924..9f77129 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -342,8 +342,6 @@ class HDP23StackAdvisor(HDP22StackAdvisor): atlas_server_host_info = self.getHostWithComponent("ATLAS", "ATLAS_SERVER", services, hosts) if include_atlas and atlas_server_host_info: - cluster_name = 'default' - putHiveSiteProperty('atlas.cluster.name', cluster_name) atlas_rest_host = atlas_server_host_info['Hosts']['host_name'] scheme = "http" metadata_port = "21000" http://git-wip-us.apache.org/repos/asf/ambari/blob/4104f2f9/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml index 99fd20f..8412fc0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml @@ -55,5 +55,13 @@ <property> <name>atlas.graph.index.search.solr.zookeeper-url</name> <description>The ZooKeeper quorum setup for Solr as comma separated value.</description> + <name>atlas.login.method</name> + <value>file</value> + <description>Atlas login method (e.g. file).</description> + </property> + <property> + <name>atlas.login.credentials.file</name> + <value>{{atlas_login_credentials_file}}</value> + <description>File path for file based login.</description> </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/4104f2f9/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 617f73d..2080c52 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -1472,6 +1472,274 @@ class TestHDP23StackAdvisor(TestCase): self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) + def test_recommendHiveConfigurations_with_atlas(self): + self.maxDiff = None + configurations = { + "yarn-site": { + "properties": { + "yarn.scheduler.minimum-allocation-mb": "256", + "yarn.scheduler.maximum-allocation-mb": "8192", + }, + } + } + clusterData = { + "cpu": 4, + "mapMemory": 3000, + "amMemory": 2000, + "reduceMemory": 2056, + "containers": 3, + "ramPerContainer": 256 + } + expected = { + 'yarn-site': { + 'properties': { + 'yarn.scheduler.minimum-allocation-mb': '256', + 'yarn.scheduler.maximum-allocation-mb': '8192' + } + }, + 'hive-env': { + 'properties': { + 'hive_exec_orc_storage_strategy': 'SPEED', + 'hive_security_authorization': 'None', + 'hive_timeline_logging_enabled': 'true', + 'hive_txn_acid': 'off' + } + }, + 'hive-site': { + 'properties': { + 'hive.server2.enable.doAs': 'true', + 'hive.server2.tez.default.queues': "queue1,queue2", + 'hive.server2.tez.initialize.default.sessions': 'false', + 'hive.server2.tez.sessions.per.default.queue': '1', + 'hive.auto.convert.join.noconditionaltask.size': '268435456', + 'hive.compactor.initiator.on': 'false', + 'hive.compactor.worker.threads': '0', + 'hive.compute.query.using.stats': 'true', + 'hive.enforce.bucketing': 'false', + 'hive.exec.dynamic.partition.mode': 'strict', + 'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook', + 'hive.exec.orc.compression.strategy': 'SPEED', + 'hive.exec.orc.default.compress': 'ZLIB', + 'hive.exec.orc.default.stripe.size': '67108864', + 'hive.exec.orc.encoding.strategy': 'SPEED', + 'hive.exec.post.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook,org.apache.atlas.hive.hook.HiveHook', + 'hive.exec.pre.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook', + 'hive.exec.reducers.bytes.per.reducer': '67108864', + 'hive.execution.engine': 'mr', + 'hive.optimize.index.filter': 'true', + 'hive.optimize.sort.dynamic.partition': 'false', + 'hive.prewarm.enabled': 'false', + 'hive.prewarm.numcontainers': '3', + 'hive.security.authorization.enabled': 'false', + 'hive.server2.use.SSL': 'false', + 'hive.stats.fetch.column.stats': 'true', + 'hive.stats.fetch.partition.stats': 'true', + 'hive.support.concurrency': 'false', + 'hive.tez.auto.reducer.parallelism': 'true', + 'hive.tez.container.size': '768', + 'hive.tez.dynamic.partition.pruning': 'true', + 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps', + 'hive.txn.manager': 'org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager', + 'hive.vectorized.execution.enabled': 'true', + 'hive.vectorized.execution.reduce.enabled': 'false', + 'hive.security.metastore.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider', + 'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory' + }, + 'property_attributes': { + 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'}, + 'hive.server2.authentication.pam.services': {'delete': 'true'}, + 'hive.server2.custom.authentication.class': {'delete': 'true'}, + 'hive.server2.authentication.kerberos.principal': {'delete': 'true'}, + 'hive.server2.authentication.kerberos.keytab': {'delete': 'true'}, + 'hive.server2.authentication.ldap.url': {'delete': 'true'}, + 'hive.server2.tez.default.queues': { + 'entries': [{'value': 'queue1', 'label': 'queue1 queue'}, {'value': 'queue2', 'label': 'queue2 queue'}] + }, + 'atlas.cluster.name': {'delete': 'true'}, + 'atlas.rest.address': {'delete': 'true'}, + 'datanucleus.rdbms.datastoreAdapterClassName': {'delete': 'true'} + } + }, + 'hiveserver2-site': { + 'properties': { + }, + 'property_attributes': { + 'hive.security.authorization.manager': {'delete': 'true'}, + 'hive.security.authenticator.manager': {'delete': 'true'} + } + } + } + services = { + "services": [ + { + "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN", + "StackServices": { + "service_name": "YARN", + "service_version": "2.6.0.2.2", + "stack_name": "HDP", + "stack_version": "2.2" + }, + "components": [ + { + "StackServiceComponents": { + "advertise_version": "false", + "cardinality": "1", + "component_category": "MASTER", + "component_name": "APP_TIMELINE_SERVER", + "display_name": "App Timeline Server", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "NODEMANAGER", + "display_name": "NodeManager", + "is_client": "false", + "is_master": "false", + "hostnames": [ + "c6403.ambari.apache.org" + ] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1-2", + "component_category": "MASTER", + "component_name": "RESOURCEMANAGER", + "display_name": "ResourceManager", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1+", + "component_category": "CLIENT", + "component_name": "YARN_CLIENT", + "display_name": "YARN Client", + "is_client": "true", + "is_master": "false", + "hostnames": [] + }, + "dependencies": [] + } + ] + }, + { + "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS", + "StackServices": { + "service_name": "ATLAS", + "service_version": "2.6.0.2.2", + "stack_name": "HDP", + "stack_version": "2.3" + }, + "components": [ + { + "StackServiceComponents": { + "advertise_version": "false", + "cardinality": "1", + "component_category": "MASTER", + "component_name": "ATLAS_SERVER", + "display_name": "Atlas Server", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + } + ] + } + ], + "configurations": { + "capacity-scheduler": { + "properties": { + "capacity-scheduler" :"yarn.scheduler.capacity.root.queues=queue1,queue2" + } + }, + "hive-env": { + "properties": { + } + }, + "hive-site": { + "properties": { + "hive.server2.authentication": "none", + "hive.server2.authentication.ldap.url": "", + "hive.server2.authentication.ldap.baseDN": "", + "hive.server2.authentication.kerberos.keytab": "", + "hive.server2.authentication.kerberos.principal": "", + "hive.server2.authentication.pam.services": "", + "hive.server2.custom.authentication.class": "", + "hive.cbo.enable": "true" + } + }, + "hiveserver2-site": { + "properties": { + "hive.security.authorization.manager": "", + "hive.security.authenticator.manager": "" + } + } + }, + "changed-configurations": [ ] + + } + hosts = { + "items" : [ + { + "href" : "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6401.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6401.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + }, + { + "href" : "/api/v1/hosts/c6402.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6402.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6402.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + }, + { + "href" : "/api/v1/hosts/c6403.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6403.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6403.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + } + ] + } + + self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + @patch('os.path.exists') @patch('os.path.isdir') @patch('os.listdir')
