Repository: hive Updated Branches: refs/heads/master dcfb1ed27 -> a977c3680
HIVE-14887. Reduce the memory used by MiniMr, MiniTez, MiniLlap tests. (Siddharth Seth, reviewed by Sergio Peña) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a977c368 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a977c368 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a977c368 Branch: refs/heads/master Commit: a977c3680246f03d8abbcb1210a78f15c68dac2d Parents: dcfb1ed Author: Siddharth Seth <[email protected]> Authored: Thu Oct 27 13:22:33 2016 -0700 Committer: Siddharth Seth <[email protected]> Committed: Thu Oct 27 13:22:33 2016 -0700 ---------------------------------------------------------------------- data/conf/llap/tez-site.xml | 5 ++++ data/conf/perf-reg/tez-site.xml | 4 +++ data/conf/tez/tez-site.xml | 4 +++ .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 7 +++--- .../org/apache/hadoop/hive/ql/QTestUtil.java | 8 +++--- pom.xml | 3 ++- .../apache/hadoop/hive/shims/Hadoop23Shims.java | 26 ++++++++++++++++++++ .../ptest2/src/main/resources/batch-exec.vm | 2 +- .../ptest2/src/main/resources/source-prep.vm | 2 +- 9 files changed, 50 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/data/conf/llap/tez-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/llap/tez-site.xml b/data/conf/llap/tez-site.xml index 6f1b9d2..62f97c5 100644 --- a/data/conf/llap/tez-site.xml +++ b/data/conf/llap/tez-site.xml @@ -5,6 +5,11 @@ <value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value> </property> + <property> + <name>tez.am.resource.memory.mb</name> + <value>128</value> + </property> + <!-- Fail fast during tests --> <property> <name>tez.am.task.max.failed.attempts</name> http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/data/conf/perf-reg/tez-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/perf-reg/tez-site.xml b/data/conf/perf-reg/tez-site.xml index 940f390..c575544 100644 --- a/data/conf/perf-reg/tez-site.xml +++ b/data/conf/perf-reg/tez-site.xml @@ -1,5 +1,9 @@ <configuration> <property> + <name>tez.am.resource.memory.mb</name> + <value>128</value> + </property> + <property> <name>tez.am.dag.scheduler.class</name> <value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value> </property> http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/data/conf/tez/tez-site.xml ---------------------------------------------------------------------- diff --git a/data/conf/tez/tez-site.xml b/data/conf/tez/tez-site.xml index 940f390..c575544 100644 --- a/data/conf/tez/tez-site.xml +++ b/data/conf/tez/tez-site.xml @@ -1,5 +1,9 @@ <configuration> <property> + <name>tez.am.resource.memory.mb</name> + <value>128</value> + </property> + <property> <name>tez.am.dag.scheduler.class</name> <value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value> </property> http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index f02f0e3..d06c869 100644 --- a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -223,7 +223,8 @@ public class MiniHS2 extends AbstractHiveService { // Initialize the execution engine based on cluster type switch (miniClusterType) { case TEZ: - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString); + // TODO: This should be making use of confDir to load configs setup for Tez, etc. + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString); break; case LLAP: if (usePortsFromConf) { @@ -231,10 +232,10 @@ public class MiniHS2 extends AbstractHiveService { } llapCluster = LlapItUtils.startAndGetMiniLlapCluster(hiveConf, null, null); - mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString); + mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString); break; case MR: - mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 4, uriString, 1); + mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 2, uriString, 1); break; default: throw new IllegalArgumentException("Unsupported cluster type " + mr); http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index a873721..bfb9645 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -654,12 +654,10 @@ public class QTestUtil { conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + "/tez-site.xml")); } - int numTrackers; + int numTrackers = 2; if (EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType)) { llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir); - numTrackers = 2; } else { - numTrackers = 4; } if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local).contains(clusterType)) { mr = shims.getLocalMiniTezCluster(conf, clusterType == MiniClusterType.llap_local); @@ -667,9 +665,9 @@ public class QTestUtil { mr = shims.getMiniTezCluster(conf, numTrackers, uriString); } } else if (clusterType == MiniClusterType.miniSparkOnYarn) { - mr = shims.getMiniSparkCluster(conf, 4, uriString, 1); + mr = shims.getMiniSparkCluster(conf, 2, uriString, 1); } else if (clusterType == MiniClusterType.mr) { - mr = shims.getMiniMrCluster(conf, 4, uriString, 1); + mr = shims.getMiniMrCluster(conf, 2, uriString, 1); } } http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 6dad4cd..3d8fa1a 100644 --- a/pom.xml +++ b/pom.xml @@ -91,6 +91,7 @@ <!-- Plugin and Plugin Dependency Versions --> <ant.contrib.version>1.0b3</ant.contrib.version> <datanucleus.maven.plugin.version>3.3.0-release</datanucleus.maven.plugin.version> + <maven.test.jvm.args>-Xmx2048m</maven.test.jvm.args> <maven.antrun.plugin.version>1.7</maven.antrun.plugin.version> <maven.assembly.plugin.version>2.3</maven.assembly.plugin.version> <maven.checkstyle.plugin.version>2.12.1</maven.checkstyle.plugin.version> @@ -1003,7 +1004,7 @@ <redirectTestOutputToFile>true</redirectTestOutputToFile> <reuseForks>false</reuseForks> <failIfNoTests>false</failIfNoTests> - <argLine>-Xmx2048m -XX:MaxPermSize=512m</argLine> + <argLine>${maven.test.jvm.args}</argLine> <additionalClasspathElements> <additionalClasspathElement>${test.conf.dir}</additionalClasspathElement> <additionalClasspathElement>${basedir}/${hive.path.to.root}/conf</additionalClasspathElement> http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java ---------------------------------------------------------------------- diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index ff760c8..e56463b 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -293,6 +293,12 @@ public class Hadoop23Shims extends HadoopShimsSecure { JobConf jConf = new JobConf(conf); jConf.set("yarn.scheduler.capacity.root.queues", "default"); jConf.set("yarn.scheduler.capacity.root.default.capacity", "100"); + jConf.setInt(MRJobConfig.MAP_MEMORY_MB, 128); + jConf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128); + jConf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128); + jConf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512); + jConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); + jConf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512); mr = new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir, null, null, jConf); } @@ -320,6 +326,9 @@ public class Hadoop23Shims extends HadoopShimsSecure { for (Map.Entry<String, String> pair: jConf) { conf.set(pair.getKey(), pair.getValue()); } + conf.setInt(MRJobConfig.MAP_MEMORY_MB, 128); + conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128); + conf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128); } } @@ -385,6 +394,16 @@ public class Hadoop23Shims extends HadoopShimsSecure { public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); + conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512); + // Overrides values from the hive/tez-site. + conf.setInt("hive.tez.container.size", 128); + conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128); + conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 24); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 10); + conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); conf.set("fs.defaultFS", nameNode); conf.set("tez.am.log.level", "DEBUG"); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); @@ -416,6 +435,13 @@ public class Hadoop23Shims extends HadoopShimsSecure { for (Map.Entry<String, String> pair: config) { conf.set(pair.getKey(), pair.getValue()); } + // Overrides values from the hive/tez-site. + conf.setInt("hive.tez.container.size", 128); + conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128); + conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 24); + conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 10); + conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); } } http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/testutils/ptest2/src/main/resources/batch-exec.vm ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/resources/batch-exec.vm b/testutils/ptest2/src/main/resources/batch-exec.vm index 00487ce..2d16ca3 100644 --- a/testutils/ptest2/src/main/resources/batch-exec.vm +++ b/testutils/ptest2/src/main/resources/batch-exec.vm @@ -35,7 +35,7 @@ then export PATH=$JAVA_HOME/bin/:$PATH fi export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${antEnvOpts}" -export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}" +export MAVEN_OPTS="-Xmx256m -Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}" export HADOOP_ROOT_LOGGER=INFO,console export HADOOP_OPTS="-Dhive.log.dir=$logDir -Dhive.query.id=hadoop -Djava.io.tmpdir=$logDir/tmp" cd $localDir/$instanceName/${repositoryName}-source || exit 1 http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/testutils/ptest2/src/main/resources/source-prep.vm ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/resources/source-prep.vm b/testutils/ptest2/src/main/resources/source-prep.vm index 0fc22be..7ad5024 100644 --- a/testutils/ptest2/src/main/resources/source-prep.vm +++ b/testutils/ptest2/src/main/resources/source-prep.vm @@ -23,7 +23,7 @@ then export PATH=$JAVA_HOME/bin/:$PATH fi export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m ${antEnvOpts}" -export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m ${mavenEnvOpts}" +export MAVEN_OPTS="-Xmx1g ${mavenEnvOpts}" cd $workingDir/ ( if [[ "$clearLibraryCache" == "true" ]]
