Author: omalley
Date: Tue Mar 8 06:01:33 2011
New Revision: 1079262
URL: http://svn.apache.org/viewvc?rev=1079262&view=rev
Log:
commit df3ff80d8f32cc14edae564d7a2b82ea6cbf3139
Author: Vinay Kumar Thota <[email protected]>
Date: Tue Feb 8 01:13:34 2011 -0800
3957917 from
+++ b/YAHOO-CHANGES.txt
+
+ [MR-2053] : Test Gridmix file pool for different input
+ file sizes based on pool minimum size.Patch available at
+ (vinayt)
Added:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java?rev=1079262&r1=1079261&r2=1079262&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixDataGeneration.java
Tue Mar 8 06:01:33 2011
@@ -91,8 +91,14 @@ public class TestGridMixDataGeneration {
inputSize+"m",
"file:///dev/null"};
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+ "=false"
+ };
int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir,
- conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
+ conf,GridMixRunMode.DATA_GENERATION, runtimeValues, otherArgs);
Assert.assertEquals("Data generation has failed.", 0 , exitCode);
checkGeneratedDataAndJobStatus(inputSize);
}
@@ -114,8 +120,15 @@ public class TestGridMixDataGeneration {
"file://" + UtilsForGridmix.getProxyUsersFile(conf),
"file:///dev/null"};
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+ "=false"
+ };
+
int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir,
- conf,GridMixRunMode.DATA_GENERATION, runtimeValues);
+ conf,GridMixRunMode.DATA_GENERATION, runtimeValues, otherArgs);
Assert.assertEquals("Data generation has failed.", 0 , exitCode);
checkGeneratedDataAndJobStatus(inputSize);
}
@@ -140,7 +153,11 @@ public class TestGridMixDataGeneration {
int bytesPerFile = 200; // 200 mb per file of data
String [] otherArgs = {
"-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE +
- "=" + (bytesPerFile * 1024 * 1024)
+ "=" + (bytesPerFile * 1024 * 1024),
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+ "=false"
};
int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir,
conf,GridMixRunMode.DATA_GENERATION, runtimeValues,otherArgs);
Added:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java?rev=1079262&view=auto
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java
(added)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/TestGridMixFilePool.java
Tue Mar 8 06:01:33 2011
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.gridmix;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.test.system.MRCluster;
+import org.apache.hadoop.mapreduce.test.system.JTClient;
+import org.apache.hadoop.mapreduce.test.system.JTProtocol;
+import org.apache.hadoop.mapred.gridmix.FilePool;
+import org.apache.hadoop.mapred.gridmix.test.system.UtilsForGridmix;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixRunMode;
+import org.apache.hadoop.mapred.gridmix.test.system.GridMixConfig;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
+import org.junit.Test;
+import java.io.IOException;
+import java.util.ArrayList;
+
+public class TestGridMixFilePool {
+ private static final Log LOG = LogFactory
+ .getLog(TestGridMixFilePool.class);
+ private static Configuration conf = new Configuration();
+ private static MRCluster cluster;
+ private static JTProtocol remoteClient;
+ private static JTClient jtClient;
+ private static Path gridmixDir;
+ private static int clusterSize;
+
+ @BeforeClass
+ public static void before() throws Exception {
+ String [] excludeExpList = {"java.net.ConnectException",
+ "java.io.IOException"};
+ cluster = MRCluster.createCluster(conf);
+ cluster.setExcludeExpList(excludeExpList);
+ cluster.setUp();
+ jtClient = cluster.getJTClient();
+ remoteClient = jtClient.getProxy();
+ clusterSize = cluster.getTTClients().size();
+ gridmixDir = new Path("herriot-gridmix");
+ UtilsForGridmix.createDirs(gridmixDir, remoteClient.getDaemonConf());
+ }
+
+ @AfterClass
+ public static void after() throws Exception {
+ UtilsForGridmix.cleanup(gridmixDir, conf);
+ cluster.tearDown();
+ }
+
+ @Test
+ public void testFilesCountAndSizesForSpecifiedFilePool() throws Exception {
+ conf = remoteClient.getDaemonConf();
+ final long inputSize = clusterSize * 200;
+ int [] fileSizesInMB = {50, 100, 400, 50, 300, 10, 60, 40, 20 ,10 , 500};
+ long targetSize = Long.MAX_VALUE;
+ final int expFileCount = clusterSize + 4;
+ String [] runtimeValues ={"LOADJOB",
+ SubmitterUserResolver.class.getName(),
+ "STRESS",
+ inputSize+"m",
+ "file:///dev/null"};
+
+ String [] otherArgs = {
+ "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE +
+ "=false",
+ "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE +
+ "=false"
+ };
+
+ int exitCode = UtilsForGridmix.runGridmixJob(gridmixDir,
+ conf,GridMixRunMode.DATA_GENERATION, runtimeValues, otherArgs);
+ Assert.assertEquals("Data generation has failed.", 0 , exitCode);
+ // create files for given sizes.
+ createFiles(new Path(gridmixDir,"input"),fileSizesInMB);
+ conf.setLong(FilePool.GRIDMIX_MIN_FILE, 100 * 1024 * 1024);
+ FilePool fpool = new FilePool(conf,new Path(gridmixDir,"input"));
+ fpool.refresh();
+ verifyFilesSizeAndCountForSpecifiedPool(expFileCount,targetSize, fpool);
+ }
+
+ private void createFiles(Path inputDir, int [] fileSizes)
+ throws Exception {
+ for (int size : fileSizes) {
+ UtilsForGridmix.createFile(size, inputDir, conf);
+ }
+ }
+
+ private void verifyFilesSizeAndCountForSpecifiedPool(int expFileCount,
+ long minFileSize, FilePool pool) throws IOException {
+ final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
+ long actFilesSize = pool.getInputFiles(minFileSize, files)/(1024 * 1024);
+ long expFilesSize = (clusterSize * 200) + 1300;
+ Assert.assertEquals("Files Size has not matched for specified pool.",
+ expFilesSize, actFilesSize);
+ int actFileCount = files.size();
+ Assert.assertEquals("File count has not matched.",
+ expFileCount, actFileCount);
+ int count = 0;
+ for (FileStatus fstat : files) {
+ String fp = fstat.getPath().toString();
+ count = count + ((fp.indexOf("datafile_") > 0)? 0 : 1);
+ }
+ Assert.assertEquals("Total folders are not matched with cluster size",
+ clusterSize, count);
+ }
+}
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java?rev=1079262&r1=1079261&r2=1079262&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/GridMixConfig.java
Tue Mar 8 06:01:33 2011
@@ -72,4 +72,16 @@ public class GridMixConfig {
*/
public static final String GRIDMIX_MINIMUM_FILE_SIZE =
"gridmix.min.file.size";
+
+ /**
+ * Gridmix compression enable
+ */
+ public static final String GRIDMIX_COMPRESSION_ENABLE =
+ "gridmix.compression-emulation.enable";
+
+ /**
+ * Gridmix distcache enable
+ */
+ public static final String GRIDMIX_DISTCACHE_ENABLE =
+ "gridmix.distributed-cache-emulation.enable";
}
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java?rev=1079262&r1=1079261&r2=1079262&view=diff
==============================================================================
---
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
(original)
+++
hadoop/mapreduce/branches/yahoo-merge/src/contrib/gridmix/src/test/system/org/apache/hadoop/mapred/gridmix/test/system/UtilsForGridmix.java
Tue Mar 8 06:01:33 2011
@@ -86,7 +86,7 @@ public class UtilsForGridmix {
"-D", GridMixConfig.GRIDMIX_LOG_MODE +
"=DEBUG",
"-D", GridMixConfig.GRIDMIX_OUTPUT_DIR +
- "=" + new Path(gridmixDir,"gridmix").toString(),
+ "=" + new Path("gridmix").toString(),
"-D", GridMixConfig.GRIDMIX_JOB_SUBMISSION_QUEUE_IN_TRACE
+ "=true",
"-D", GridMixConfig.GRIDMIX_JOB_TYPE
@@ -103,22 +103,23 @@ public class UtilsForGridmix {
classArgs = new String[]{
"-generate", values[3],
"-users", values[4],
- new Path(gridmixDir,"input").toString(),
+ gridmixDir.toString(),
values[5]};
} else if (gridmixRunMode == GridMixRunMode.DATA_GENERATION ||
gridmixRunMode == GridMixRunMode.DATA_GENERATION_AND_RUN_GRIDMIX){
classArgs = new String[]{
- "-generate", values[3], new Path(gridmixDir,"input").toString(),
+ "-generate", values[3],
+ gridmixDir.toString(),
values[4]};
} else if(gridmixRunMode == GridMixRunMode.RUN_GRIDMIX
&& values[1].indexOf("RoundRobinUserResolver") > 0) {
classArgs = new String[]{
"-users", values[3],
- new Path(gridmixDir,"input").toString(),
+ gridmixDir.toString(),
values[4]};
} else {
classArgs = new String[]{
- new Path(gridmixDir,"input").toString(),values[3]};
+ gridmixDir.toString(),values[3]};
}
String [] args = new String [runtimeArgs.length +
