Repository: ambari Updated Branches: refs/heads/trunk fddbaf4dd -> 211c78bbb
AMBARI-10714 - Separate memory configuration for Hive CLI vs HiveServer2 (metastore part) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/211c78bb Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/211c78bb Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/211c78bb Branch: refs/heads/trunk Commit: 211c78bbbae794a0c5d7fdb04327bf2960bb96ea Parents: fddbaf4 Author: Artem Baranchuk <[email protected]> Authored: Mon May 18 17:31:37 2015 +0300 Committer: Artem Baranchuk <[email protected]> Committed: Tue May 19 14:24:23 2015 +0300 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog210.java | 33 ++++++++++++++++++++ .../HIVE/0.12.0.2.0/configuration/hive-env.xml | 13 +++++++- .../0.12.0.2.0/package/scripts/params_linux.py | 2 ++ .../services/HIVE/configuration/hive-env.xml | 5 +++ .../server/upgrade/UpgradeCatalog210Test.java | 9 +++++- 5 files changed, 60 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/211c78bb/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java index b10d3a5..69c82ef 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java @@ -873,6 +873,39 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog { // Initialize all default widgets and widget layouts initializeClusterAndServiceWidgets(); + + addMissingConfigs(); + } + + protected void addMissingConfigs() throws AmbariException { + updateHiveConfigs(); + } + + protected void updateHiveConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance( + AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + + if (clusters != null) { + Map<String, Cluster> clusterMap = clusters.getClusters(); + Map<String, String> prop = new HashMap<String, String>(); + + if (clusterMap != null && !clusterMap.isEmpty()) { + for (final Cluster cluster : clusterMap.values()) { + //hive metastore and client_heapsize are added for HDP2, we should check if it exists and not add it for HDP1 + if(cluster.getDesiredConfigByType("hive-env") != null) { + Map<String, String> hiveProps = new HashMap<String, String>(); + if (!cluster.getDesiredConfigByType("hive-env").getProperties().containsKey("hive.client.heapsize")) { + hiveProps.put("hive.client.heapsize", "512m"); + } + if (!cluster.getDesiredConfigByType("hive-env").getProperties().containsKey("hive.metastore.heapsize")) { + hiveProps.put("hive.metastore.heapsize", "1024m"); + } + updateConfigurationPropertiesForCluster(cluster, "hive-env", hiveProps, false, true); + } + } + } + } } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/211c78bb/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml index 923a9ba..db2c927 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml @@ -28,6 +28,12 @@ </property> <property> + <name>hive.metastore.heapsize</name> + <value>1024</value> + <description>Hive Metastore Java heap size</description> + </property> + + <property> <name>hive_database_type</name> <value>mysql</value> <description>Default HIVE DB type.</description> @@ -106,7 +112,12 @@ # The heap size of the jvm stared by hive shell script can be controlled via: -export HADOOP_HEAPSIZE="{{hive_heapsize}}" +if [ "$SERVICE" = "metastore" ]; then + export HADOOP_HEAPSIZE="{{hive_metastore_heapsize}}" +else + export HADOOP_HEAPSIZE="{{hive_heapsize}}" +fi + export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS" # Larger heap size may be required when running queries over large number of files or partitions. http://git-wip-us.apache.org/repos/asf/ambari/blob/211c78bb/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 7254f3c..4396401 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -253,6 +253,8 @@ if 'role' in config and config['role'] in ["HIVE_SERVER", "HIVE_METASTORE"]: else: hive_heapsize = config['configurations']['hive-env']['hive.client.heapsize'] +hive_metastore_heapsize = config['configurations']['hive-env']['hive.metastore.heapsize'] + java64_home = config['hostLevelParams']['java_home'] ##### MYSQL http://git-wip-us.apache.org/repos/asf/ambari/blob/211c78bb/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml index f542bda..7d1aa2d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml @@ -26,6 +26,11 @@ <deleted>true</deleted> </property> + <property> + <name>hive.metastore.heapsize</name> + <deleted>true</deleted> + </property> + <!-- hive-env.sh --> <property> <name>content</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/211c78bb/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java index 13435e7..ac85f0d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java @@ -151,9 +151,13 @@ public class UpgradeCatalog210Test { Method initializeClusterAndServiceWidgets = UpgradeCatalog210.class.getDeclaredMethod("initializeClusterAndServiceWidgets"); + Method addMissingConfigs = + UpgradeCatalog210.class.getDeclaredMethod("addMissingConfigs"); + UpgradeCatalog210 upgradeCatalog210 = createMockBuilder(UpgradeCatalog210.class) .addMockedMethod(addNewConfigurationsFromXml) - .addMockedMethod(initializeClusterAndServiceWidgets).createMock(); + .addMockedMethod(initializeClusterAndServiceWidgets) + .addMockedMethod(addMissingConfigs).createMock(); upgradeCatalog210.addNewConfigurationsFromXml(); expectLastCall().once(); @@ -161,6 +165,9 @@ public class UpgradeCatalog210Test { upgradeCatalog210.initializeClusterAndServiceWidgets(); expectLastCall().once(); + upgradeCatalog210.addMissingConfigs(); + expectLastCall().once(); + replay(upgradeCatalog210); upgradeCatalog210.executeDMLUpdates();
