http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml new file mode 100644 index 0000000..4bd1a19 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/TEZ/configuration/tez-site.xml @@ -0,0 +1,28 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration supports_final="true"> + <property> + <name>tez.task.launch.cmd-opts</name> + <value>-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{{heap_dump_opts}}</value> + <description>Java options for tasks. The Xmx value is derived based on tez.task.resource.memory.mb and is 80% of this value by default. + Used only if the value is not specified explicitly by the DAG definition. + </description> + <on-ambari-upgrade add="false"/> + </property> +</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py index 562444b..9072dd0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py @@ -18,6 +18,7 @@ limitations under the License. """ from resource_management.core.logger import Logger import json +import re from resource_management.libraries.functions import format @@ -30,7 +31,8 @@ class HDP26StackAdvisor(HDP25StackAdvisor): parentRecommendConfDict = super(HDP26StackAdvisor, self).getServiceConfigurationRecommenderDict() childRecommendConfDict = { "DRUID": self.recommendDruidConfigurations, - "ATLAS": self.recommendAtlasConfigurations + "ATLAS": self.recommendAtlasConfigurations, + "TEZ": self.recommendTezConfigurations } parentRecommendConfDict.update(childRecommendConfDict) return parentRecommendConfDict @@ -225,3 +227,24 @@ class HDP26StackAdvisor(HDP25StackAdvisor): "druid.processing.numThreads")} ] return self.toConfigurationValidationProblems(validationItems, "druid-broker") + + def recommendTezConfigurations(self, configurations, clusterData, services, hosts): + super(HDP26StackAdvisor, self).recommendTezConfigurations(configurations, clusterData, services, hosts) + putTezProperty = self.putProperty(configurations, "tez-site") + + # TEZ JVM options + jvmGCParams = "-XX:+UseParallelGC" + if "ambari-server-properties" in services and "java.home" in services["ambari-server-properties"]: + # JDK8 needs different parameters + match = re.match(".*\/jdk(1\.\d+)[\-\_\.][^/]*$", services["ambari-server-properties"]["java.home"]) + if match and len(match.groups()) > 0: + # Is version >= 1.8 + versionSplits = re.split("\.", match.group(1)) + if versionSplits and len(versionSplits) > 1 and int(versionSplits[0]) > 0 and int(versionSplits[1]) > 7: + jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB" + tez_jvm_opts = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA " + # Append 'jvmGCParams' and 'Heap Dump related option' (({{heap_dump_opts}}) Expanded while writing the + # configurations at start/restart time). + tez_jvm_updated_opts = tez_jvm_opts + jvmGCParams + "{{heap_dump_opts}}" + putTezProperty('tez.task.launch.cmd-opts', tez_jvm_updated_opts) + Logger.info("Updated 'tez-site' config 'tez.task.launch.cmd-opts' as : {0}".format(tez_jvm_updated_opts)) http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java index d4f2e8b..9eb4db9 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java @@ -18,31 +18,14 @@ package org.apache.ambari.server.upgrade; -import javax.persistence.EntityManager; +import com.google.common.collect.Maps; +import com.google.gson.Gson; +import com.google.inject.Binder; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Module; +import com.google.inject.Provider; import junit.framework.Assert; -import static org.easymock.EasyMock.anyObject; -import static org.easymock.EasyMock.anyString; -import static org.easymock.EasyMock.capture; -import static org.easymock.EasyMock.createMockBuilder; -import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.createStrictMock; -import static org.easymock.EasyMock.eq; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.expectLastCall; -import static org.easymock.EasyMock.newCapture; -import static org.easymock.EasyMock.replay; -import static org.easymock.EasyMock.reset; -import static org.easymock.EasyMock.verify; -import static org.junit.Assert.assertTrue; - -import java.lang.reflect.Method; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.apache.ambari.server.actionmanager.ActionManager; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; @@ -62,13 +45,29 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import com.google.common.collect.Maps; -import com.google.gson.Gson; -import com.google.inject.Binder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.Provider; +import javax.persistence.EntityManager; +import java.lang.reflect.Method; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; +import static org.easymock.EasyMock.capture; +import static org.easymock.EasyMock.createMockBuilder; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.createStrictMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.newCapture; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; +import static org.easymock.EasyMock.verify; +import static org.junit.Assert.assertTrue; /** * {@link UpgradeCatalog250} unit tests. @@ -216,10 +215,14 @@ public class UpgradeCatalog250Test { Method updateTablesForZeppelinViewRemoval = UpgradeCatalog250.class.getDeclaredMethod("updateTablesForZeppelinViewRemoval"); Method updateAtlasConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateAtlasConfigs"); Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml"); + Method updateHIVEInteractiveConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateHIVEInteractiveConfigs"); + Method updateTEZInteractiveConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateTEZInteractiveConfigs"); UpgradeCatalog250 upgradeCatalog250 = createMockBuilder(UpgradeCatalog250.class) .addMockedMethod(updateAmsConfigs) .addMockedMethod(updateKafkaConfigs) + .addMockedMethod(updateHIVEInteractiveConfigs) + .addMockedMethod(updateTEZInteractiveConfigs) .addMockedMethod(updateHiveLlapConfigs) .addMockedMethod(updateTablesForZeppelinViewRemoval) .addMockedMethod(updateAtlasConfigs) @@ -236,6 +239,12 @@ public class UpgradeCatalog250Test { upgradeCatalog250.updateKafkaConfigs(); expectLastCall().once(); + upgradeCatalog250.updateHIVEInteractiveConfigs(); + expectLastCall().once(); + + upgradeCatalog250.updateTEZInteractiveConfigs(); + expectLastCall().once(); + upgradeCatalog250.updateHiveLlapConfigs(); expectLastCall().once(); http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 9d8ba08..02d6c80 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1548,12 +1548,11 @@ class TestHDP22StackAdvisor(TestCase): expected["hive-site"]["properties"]["hive.stats.fetch.column.stats"]="false" expected["hive-site"]["properties"]["hive.security.authorization.enabled"]="true" expected["hive-site"]["properties"]["hive.server2.enable.doAs"]="false" - expected["hive-site"]["properties"]["hive.security.metastore.authorization.manager"]=\ - "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly" + expected["hive-site"]["properties"]["hive.security.metastore.authorization.manager"]="org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider,org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly" expected["hiveserver2-site"]["properties"]["hive.security.authorization.enabled"]="true" expected["hiveserver2-site"]["properties"]["hive.security.authorization.manager"]="org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory" expected["hiveserver2-site"]["properties"]["hive.security.authenticator.manager"]="org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator" - expected["hiveserver2-site"]["properties"]["hive.conf.restricted.list"]="hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role" + expected["hiveserver2-site"]["properties"]["hive.conf.restricted.list"]="hive.security.authenticator.manager,hive.security.authorization.manager,hive.security.metastore.authorization.manager,hive.security.metastore.authenticator.manager,hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.security.authorization.enabled" self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) @@ -1646,7 +1645,7 @@ class TestHDP22StackAdvisor(TestCase): expected["hiveserver2-site"]["properties"]["hive.security.authenticator.manager"] = "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator" expected["hiveserver2-site"]["properties"]["hive.security.authorization.manager"] = "com.xasecure.authorization.hive.authorizer.XaSecureHiveAuthorizerFactory" expected["hiveserver2-site"]["properties"]["hive.security.authorization.enabled"] = "true" - expected["hiveserver2-site"]["properties"]["hive.conf.restricted.list"]="hive.security.authorization.enabled,hive.security.authorization.manager,hive.security.authenticator.manager" + expected["hiveserver2-site"]["properties"]["hive.conf.restricted.list"]="hive.security.authenticator.manager,hive.security.authorization.manager,hive.security.metastore.authorization.manager,hive.security.metastore.authenticator.manager,hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.security.authorization.enabled" self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations['hiveserver2-site'], expected["hiveserver2-site"]) http://git-wip-us.apache.org/repos/asf/ambari/blob/8552345d/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py index 3221d5d..e00668a 100644 --- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py @@ -384,6 +384,8 @@ class TestHiveServerInteractive(RMFTestCase): hive_site_conf['hive.exec.post.hooks'] = 'a,b,org.apache.hadoop.hive.ql.hooks.ATSHook' del hive_site_conf['hive.enforce.bucketing'] del hive_site_conf['hive.enforce.sorting'] + del hive_site_conf['hive.llap.io.memory.size'] + hive_site_conf['hive.llap.io.memory.size'] = 357564416L hiveserver2_site_conf = {} hiveserver2_site_conf.update(self.getConfig()['configurations']['hiveserver2-site'])
