Author: suresh
Date: Fri Nov 30 19:58:09 2012
New Revision: 1415815
URL: http://svn.apache.org/viewvc?rev=1415815&view=rev
Log:
Merging trunk into branch-trunk-win
Added:
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java
- copied unchanged from r1415786,
hadoop/common/trunk/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
- copied unchanged from r1415786,
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
Modified:
hadoop/common/branches/branch-trunk-win/ (props changed)
hadoop/common/branches/branch-trunk-win/dev-support/test-patch.sh
hadoop/common/branches/branch-trunk-win/hadoop-project/pom.xml
hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/apt/index.apt.vm
hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/site.xml
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
hadoop/common/branches/branch-trunk-win/pom.xml
Propchange: hadoop/common/branches/branch-trunk-win/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk:r1407223-1415786
Modified: hadoop/common/branches/branch-trunk-win/dev-support/test-patch.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/dev-support/test-patch.sh?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/dev-support/test-patch.sh (original)
+++ hadoop/common/branches/branch-trunk-win/dev-support/test-patch.sh Fri Nov
30 19:58:09 2012
@@ -710,7 +710,7 @@ runTests () {
ordered_modules="$ordered_modules $module"
fi
done
- if [ -n $hdfs_modules ]; then
+ if [ -n "$hdfs_modules" ]; then
ordered_modules="$ordered_modules $hdfs_modules"
if [[ $building_common -eq 0 ]]; then
echo " Building hadoop-common with -Pnative in order to provide \
Modified: hadoop/common/branches/branch-trunk-win/hadoop-project/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-project/pom.xml?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-project/pom.xml Fri Nov 30
19:58:09 2012
@@ -703,11 +703,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.3.1</version>
- <configuration>
- <excludes>
- <exclude>mrapp-generated-classpath</exclude>
- </excludes>
- </configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -803,21 +798,6 @@
</executions>
</plugin>
<plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>build-classpath</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>build-classpath</goal>
- </goals>
- <configuration>
- <outputFile>target/classes/mrapp-generated-classpath</outputFile>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/apt/index.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/apt/index.apt.vm?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/apt/index.apt.vm
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/apt/index.apt.vm
Fri Nov 30 19:58:09 2012
@@ -34,7 +34,7 @@ Apache Hadoop ${project.version}
Namenodes.
More details are available in the
- {{{./hadoop-yarn/hadoop-yarn-site/Federation.html}HDFS Federation}}
+ {{{./hadoop-project-dist/hadoop-hdfs/Federation.html}HDFS Federation}}
document.
* {MapReduce NextGen aka YARN aka MRv2}
@@ -65,9 +65,9 @@ Getting Started
The Hadoop documentation includes the information you need to get started
using
Hadoop. Begin with the
- {{{./hadoop-yarn/hadoop-yarn-site/SingleCluster.html}Single Node Setup}}
which
+ {{{./hadoop-project-dist/hadoop-common/SingleCluster.html}Single Node
Setup}} which
shows you how to set up a single-node Hadoop installation. Then move on to
the
- {{{./hadoop-yarn/hadoop-yarn-site/ClusterSetup.html}Cluster Setup}} to learn
how
+ {{{./hadoop-project-dist/hadoop-common/ClusterSetup.html}Cluster Setup}} to
learn how
to set up a multi-node Hadoop installation.
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/site.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/site.xml?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/site.xml
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-project/src/site/site.xml
Fri Nov 30 19:58:09 2012
@@ -48,34 +48,36 @@
<menu name="Common" inherit="top">
<item name="Overview" href="index.html"/>
- <item name="Single Node Setup"
href="hadoop-yarn/hadoop-yarn-site/SingleCluster.html"/>
- <item name="Cluster Setup"
href="hadoop-yarn/hadoop-yarn-site/ClusterSetup.html"/>
- <item name="Hadoop Commands"
href="hadoop-project-dist/hadoop-common/commands_manual.html"/>
+ <item name="Single Node Setup"
href="hadoop-project-dist/hadoop-common/SingleCluster.html"/>
+ <item name="Cluster Setup"
href="hadoop-project-dist/hadoop-common/ClusterSetup.html"/>
+ <item name="CLI Mini Cluster"
href="hadoop-project-dist/hadoop-common/CLIMiniCluster.html"/>
</menu>
<menu name="HDFS" inherit="top">
- <item name="High Availability With QJM"
href="hadoop-yarn/hadoop-yarn-site/HDFSHighAvailabilityWithQJM.html"/>
- <item name="High Availability With NFS"
href="hadoop-yarn/hadoop-yarn-site/HDFSHighAvailabilityWithNFS.html"/>
- <item name="Federation"
href="hadoop-yarn/hadoop-yarn-site/Federation.html"/>
- <item name="WebHDFS REST API"
href="hadoop-yarn/hadoop-yarn-site/WebHDFS.html"/>
+ <item name="High Availability With QJM"
href="hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithQJM.html"/>
+ <item name="High Availability With NFS"
href="hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithNFS.html"/>
+ <item name="Federation"
href="hadoop-project-dist/hadoop-hdfs/Federation.html"/>
+ <item name="WebHDFS REST API"
href="hadoop-project-dist/hadoop-hdfs/WebHDFS.html"/>
<item name="HttpFS Gateway" href="hadoop-hdfs-httpfs/index.html"/>
- <item name="HDFS User Guide"
href="hadoop-project-dist/hadoop-hdfs/hdfs_user_guide.html"/>
</menu>
- <menu name="Yarn/MapReduce" inherit="top">
+ <menu name="MapReduce" inherit="top">
+ <item name="Encrypted Shuffle"
href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/EncryptedShuffle.html"/>
+ </menu>
+
+ <menu name="YARN" inherit="top">
<item name="YARN Architecture"
href="hadoop-yarn/hadoop-yarn-site/YARN.html"/>
- <item name="Writing Yarn Applications"
href="hadoop-yarn/hadoop-yarn-site/WritingYarnApplications.html"/>
+ <item name="Writing YARN Applications"
href="hadoop-yarn/hadoop-yarn-site/WritingYarnApplications.html"/>
<item name="Capacity Scheduler"
href="hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html"/>
<item name="Web Application Proxy"
href="hadoop-yarn/hadoop-yarn-site/WebApplicationProxy.html"/>
- <item name="Encrypted Shuffle"
href="hadoop-yarn/hadoop-yarn-site/EncryptedShuffle.html"/>
- <item name="Yarn Commands"
href="hadoop-yarn/hadoop-yarn-site/YarnCommands.html"/>
+ <item name="YARN Commands"
href="hadoop-yarn/hadoop-yarn-site/YarnCommands.html"/>
</menu>
- <menu name="YARN REST API's" inherit="top">
+ <menu name="YARN REST APIs" inherit="top">
<item name="Introduction"
href="hadoop-yarn/hadoop-yarn-site/WebServicesIntro.html"/>
<item name="Resource Manager"
href="hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html"/>
<item name="Node Manager"
href="hadoop-yarn/hadoop-yarn-site/NodeManagerRest.html"/>
- <item name="Mapreduce Application Master"
href="hadoop-yarn/hadoop-yarn-site/MapredAppMasterRest.html"/>
+ <item name="MR Application Master"
href="hadoop-yarn/hadoop-yarn-site/MapredAppMasterRest.html"/>
<item name="History Server"
href="hadoop-yarn/hadoop-yarn-site/HistoryServerRest.html"/>
</menu>
@@ -97,8 +99,8 @@
<menu name="Configuration" inherit="top">
<item name="core-default.xml"
href="hadoop-project-dist/hadoop-common/core-default.xml"/>
<item name="hdfs-default.xml"
href="hadoop-project-dist/hadoop-hdfs/hdfs-default.xml"/>
- <item name="yarn-default.xml"
href="hadoop-yarn/hadoop-yarn-common/yarn-default.xml"/>
<item name="mapred-default.xml"
href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml"/>
+ <item name="yarn-default.xml"
href="hadoop-yarn/hadoop-yarn-common/yarn-default.xml"/>
<item name="Deprecated Properties"
href="hadoop-project-dist/hadoop-common/DeprecatedProperties.html"/>
</menu>
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
Fri Nov 30 19:58:09 2012
@@ -27,8 +27,6 @@ import java.util.Collections;
import java.util.List;
import java.util.StringTokenizer;
-import junit.framework.TestCase;
-
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
@@ -38,111 +36,117 @@ import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.ToolRunner;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
/**
* test {@link HadoopArchives}
*/
-public class TestHadoopArchives extends TestCase {
+public class TestHadoopArchives {
- public static final String HADOOP_ARCHIVES_JAR =
JarFinder.getJar(HadoopArchives.class);
+ public static final String HADOOP_ARCHIVES_JAR = JarFinder
+ .getJar(HadoopArchives.class);
{
- ((Log4JLogger)LogFactory.getLog(org.apache.hadoop.security.Groups.class)
- ).getLogger().setLevel(Level.ERROR);
- ((Log4JLogger)org.apache.hadoop.ipc.Server.LOG
- ).getLogger().setLevel(Level.ERROR);
- ((Log4JLogger)org.apache.hadoop.util.AsyncDiskService.LOG
- ).getLogger().setLevel(Level.ERROR);
+ ((Log4JLogger) LogFactory.getLog(org.apache.hadoop.security.Groups.class))
+ .getLogger().setLevel(Level.ERROR);
+
}
private static final String inputDir = "input";
private Path inputPath;
private MiniDFSCluster dfscluster;
- private MiniMRCluster mapred;
+
+ private Configuration conf;
private FileSystem fs;
private Path archivePath;
-
- static private Path createFile(Path dir, String filename, FileSystem fs
- ) throws IOException {
+
+ static private Path createFile(Path dir, String filename, FileSystem fs)
+ throws IOException {
final Path f = new Path(dir, filename);
- final FSDataOutputStream out = fs.create(f);
+ final FSDataOutputStream out = fs.create(f);
out.write(filename.getBytes());
out.close();
return f;
}
-
- protected void setUp() throws Exception {
- super.setUp();
- dfscluster = new MiniDFSCluster(new Configuration(), 2, true, null);
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ conf.set(CapacitySchedulerConfiguration.PREFIX
+ + CapacitySchedulerConfiguration.ROOT + "."
+ + CapacitySchedulerConfiguration.QUEUES, "default");
+ conf.set(CapacitySchedulerConfiguration.PREFIX
+ + CapacitySchedulerConfiguration.ROOT + ".default."
+ + CapacitySchedulerConfiguration.CAPACITY, "100");
+ dfscluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true)
+ .build();
+
fs = dfscluster.getFileSystem();
- mapred = new MiniMRCluster(2, fs.getUri().toString(), 1);
- inputPath = new Path(fs.getHomeDirectory(), inputDir);
+ inputPath = new Path(fs.getHomeDirectory(), inputDir);
archivePath = new Path(fs.getHomeDirectory(), "archive");
fs.mkdirs(inputPath);
createFile(inputPath, "a", fs);
createFile(inputPath, "b", fs);
createFile(inputPath, "c", fs);
}
-
- protected void tearDown() throws Exception {
+
+ @After
+ public void tearDown() throws Exception {
try {
- if (mapred != null) {
- mapred.shutdown();
+ if (dfscluster != null) {
+ dfscluster.shutdown();
}
if (dfscluster != null) {
dfscluster.shutdown();
}
- } catch(Exception e) {
+ } catch (Exception e) {
System.err.println(e);
}
- super.tearDown();
}
-
-
+
+ @Test
public void testRelativePath() throws Exception {
fs.delete(archivePath, true);
final Path sub1 = new Path(inputPath, "dir1");
fs.mkdirs(sub1);
createFile(sub1, "a", fs);
- final Configuration conf = mapred.createJobConf();
final FsShell shell = new FsShell(conf);
final List<String> originalPaths = lsr(shell, "input");
System.out.println("originalPath: " + originalPaths);
final URI uri = fs.getUri();
- final String prefix = "har://hdfs-" + uri.getHost() +":" + uri.getPort()
+ final String prefix = "har://hdfs-" + uri.getHost() + ":" + uri.getPort()
+ archivePath.toUri().getPath() + Path.SEPARATOR;
{
final String harName = "foo.har";
- final String[] args = {
- "-archiveName",
- harName,
- "-p",
- "input",
- "*",
- "archive"
- };
- System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH,
HADOOP_ARCHIVES_JAR);
- final HadoopArchives har = new HadoopArchives(mapred.createJobConf());
- assertEquals(0, ToolRunner.run(har, args));
+ final String[] args = { "-archiveName", harName, "-p", "input", "*",
+ "archive" };
+ System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH,
+ HADOOP_ARCHIVES_JAR);
+ final HadoopArchives har = new HadoopArchives(conf);
+ Assert.assertEquals(0, ToolRunner.run(har, args));
- //compare results
+ // compare results
final List<String> harPaths = lsr(shell, prefix + harName);
- assertEquals(originalPaths, harPaths);
+ Assert.assertEquals(originalPaths, harPaths);
}
}
-
+
+@Test
public void testPathWithSpaces() throws Exception {
fs.delete(archivePath, true);
- //create files/directories with spaces
+ // create files/directories with spaces
createFile(inputPath, "c c", fs);
final Path sub1 = new Path(inputPath, "sub 1");
fs.mkdirs(sub1);
@@ -154,42 +158,36 @@ public class TestHadoopArchives extends
final Path sub2 = new Path(inputPath, "sub 1 with suffix");
fs.mkdirs(sub2);
createFile(sub2, "z", fs);
- final Configuration conf = mapred.createJobConf();
+
final FsShell shell = new FsShell(conf);
final String inputPathStr = inputPath.toUri().getPath();
- System.out.println("inputPathStr = " + inputPathStr);
final List<String> originalPaths = lsr(shell, inputPathStr);
final URI uri = fs.getUri();
- final String prefix = "har://hdfs-" + uri.getHost() +":" + uri.getPort()
+ final String prefix = "har://hdfs-" + uri.getHost() + ":" + uri.getPort()
+ archivePath.toUri().getPath() + Path.SEPARATOR;
- {//Enable space replacement
+ {// Enable space replacement
final String harName = "foo.har";
- final String[] args = {
- "-archiveName",
- harName,
- "-p",
- inputPathStr,
- "*",
- archivePath.toString()
- };
- System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH,
HADOOP_ARCHIVES_JAR);
- final HadoopArchives har = new HadoopArchives(mapred.createJobConf());
- assertEquals(0, ToolRunner.run(har, args));
+ final String[] args = { "-archiveName", harName, "-p", inputPathStr, "*",
+ archivePath.toString() };
+ System.setProperty(HadoopArchives.TEST_HADOOP_ARCHIVES_JAR_PATH,
+ HADOOP_ARCHIVES_JAR);
+ final HadoopArchives har = new HadoopArchives(conf);
+ Assert.assertEquals(0, ToolRunner.run(har, args));
- //compare results
+ // compare results
final List<String> harPaths = lsr(shell, prefix + harName);
- assertEquals(originalPaths, harPaths);
+ Assert.assertEquals(originalPaths, harPaths);
}
}
- private static List<String> lsr(final FsShell shell, String dir
- ) throws Exception {
+ private static List<String> lsr(final FsShell shell, String dir)
+ throws Exception {
System.out.println("lsr root=" + dir);
- final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
final PrintStream out = new PrintStream(bytes);
final PrintStream oldOut = System.out;
final PrintStream oldErr = System.err;
@@ -197,7 +195,7 @@ public class TestHadoopArchives extends
System.setErr(out);
final String results;
try {
- assertEquals(0, shell.run(new String[]{"-lsr", dir}));
+ Assert.assertEquals(0, shell.run(new String[] { "-lsr", dir }));
results = bytes.toString();
} finally {
IOUtils.closeStream(out);
@@ -206,13 +204,13 @@ public class TestHadoopArchives extends
}
System.out.println("lsr results:\n" + results);
String dirname = dir;
- if (dir.lastIndexOf(Path.SEPARATOR) != -1 ) {
+ if (dir.lastIndexOf(Path.SEPARATOR) != -1) {
dirname = dir.substring(dir.lastIndexOf(Path.SEPARATOR));
}
final List<String> paths = new ArrayList<String>();
- for(StringTokenizer t = new StringTokenizer(results, "\n");
- t.hasMoreTokens(); ) {
+ for (StringTokenizer t = new StringTokenizer(results, "\n"); t
+ .hasMoreTokens();) {
final String s = t.nextToken();
final int i = s.indexOf(dirname);
if (i >= 0) {
@@ -220,7 +218,8 @@ public class TestHadoopArchives extends
}
}
Collections.sort(paths);
- System.out.println("lsr paths = " + paths.toString().replace(", ", ",\n
"));
+ System.out
+ .println("lsr paths = " + paths.toString().replace(", ", ",\n "));
return paths;
}
}
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
Fri Nov 30 19:58:09 2012
@@ -274,7 +274,7 @@ public class DistCpOptions {
* @param maxMaps - Number of maps
*/
public void setMaxMaps(int maxMaps) {
- this.maxMaps = maxMaps;
+ this.maxMaps = Math.max(maxMaps, 1);
}
/** Get the map bandwidth in MB
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
Fri Nov 30 19:58:09 2012
@@ -131,8 +131,8 @@ public class TestCopyListing extends Sim
fs = FileSystem.get(getConf());
List<Path> srcPaths = new ArrayList<Path>();
srcPaths.add(new Path("/tmp/in/*/*"));
- TestDistCpUtils.createFile(fs, "/tmp/in/1.txt");
- TestDistCpUtils.createFile(fs, "/tmp/in/src/1.txt");
+ TestDistCpUtils.createFile(fs, "/tmp/in/src1/1.txt");
+ TestDistCpUtils.createFile(fs, "/tmp/in/src2/1.txt");
Path target = new Path("/tmp/out");
Path listingFile = new Path("/tmp/list");
DistCpOptions options = new DistCpOptions(srcPaths, target);
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
Fri Nov 30 19:58:09 2012
@@ -275,6 +275,13 @@ public class TestOptionsParser {
"hdfs://localhost:8020/target/"});
Assert.assertEquals(options.getMaxMaps(), 1);
+ options = OptionsParser.parse(new String[] {
+ "-m",
+ "0",
+ "hdfs://localhost:8020/source/first",
+ "hdfs://localhost:8020/target/"});
+ Assert.assertEquals(options.getMaxMaps(), 1);
+
try {
OptionsParser.parse(new String[] {
"-m",
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
Fri Nov 30 19:58:09 2012
@@ -43,21 +43,19 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSCluster.Builder;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.tools.DistCpV1;
import org.apache.hadoop.util.ToolRunner;
import org.apache.log4j.Level;
-import org.junit.Ignore;
/**
* A JUnit test for copying files recursively.
*/
-@Ignore
+
public class TestCopyFiles extends TestCase {
{
((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.StateChange")
@@ -738,20 +736,22 @@ public class TestCopyFiles extends TestC
public void testMapCount() throws Exception {
String namenode = null;
MiniDFSCluster dfs = null;
- MiniMRCluster mr = null;
+ MiniDFSCluster mr = null;
try {
Configuration conf = new Configuration();
- dfs = new MiniDFSCluster(conf, 3, true, null);
+
+ dfs= new
MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).build();
+
FileSystem fs = dfs.getFileSystem();
final FsShell shell = new FsShell(conf);
namenode = fs.getUri().toString();
- mr = new MiniMRCluster(3, namenode, 1);
MyFile[] files = createFiles(fs.getUri(), "/srcdat");
long totsize = 0;
for (MyFile f : files) {
totsize += f.getSize();
}
- Configuration job = mr.createJobConf();
+
+ Configuration job = new JobConf(conf);
job.setLong("distcp.bytes.per.map", totsize / 3);
ToolRunner.run(new DistCpV1(job),
new String[] {"-m", "100",
@@ -766,8 +766,7 @@ public class TestCopyFiles extends TestC
System.out.println(execCmd(shell, "-lsr", logdir));
FileStatus[] logs = fs.listStatus(new Path(logdir));
// rare case where splits are exact, logs.length can be 4
- assertTrue("Unexpected map count, logs.length=" + logs.length,
- logs.length == 5 || logs.length == 4);
+ assertTrue( logs.length == 2);
deldir(fs, "/destdat");
deldir(fs, "/logs");
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
---
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
(original)
+++
hadoop/common/branches/branch-trunk-win/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
Fri Nov 30 19:58:09 2012
@@ -22,8 +22,6 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
import java.util.Random;
import org.apache.commons.logging.LogFactory;
@@ -39,10 +37,10 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.mapred.MiniMRClientClusterFactory;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.log4j.Level;
-import org.junit.Ignore;
-@Ignore
+
public class TestDistCh extends junit.framework.TestCase {
{
((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.StateChange")
@@ -52,7 +50,8 @@ public class TestDistCh extends junit.fr
}
static final Long RANDOM_NUMBER_GENERATOR_SEED = null;
-
+ static final FsPermission UMASK = FsPermission.createImmutable((short)0111);
+
private static final Random RANDOM = new Random();
static {
final long seed = RANDOM_NUMBER_GENERATOR_SEED == null?
@@ -65,7 +64,7 @@ public class TestDistCh extends junit.fr
new Path(System.getProperty("test.build.data","/tmp")
).toString().replace(' ', '+');
- static final int NUN_SUBS = 5;
+ static final int NUN_SUBS = 7;
static class FileTree {
private final FileSystem fs;
@@ -127,9 +126,12 @@ public class TestDistCh extends junit.fr
public void testDistCh() throws Exception {
final Configuration conf = new Configuration();
- final MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
+
+
conf.set(CapacitySchedulerConfiguration.PREFIX+CapacitySchedulerConfiguration.ROOT+"."+CapacitySchedulerConfiguration.QUEUES,
"default");
+
conf.set(CapacitySchedulerConfiguration.PREFIX+CapacitySchedulerConfiguration.ROOT+".default."+CapacitySchedulerConfiguration.CAPACITY,
"100");
+ final MiniDFSCluster cluster= new
MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).build();
+
final FileSystem fs = cluster.getFileSystem();
- final MiniMRCluster mr = new MiniMRCluster(2, fs.getUri().toString(), 1);
final FsShell shell = new FsShell(conf);
try {
@@ -138,37 +140,36 @@ public class TestDistCh extends junit.fr
runLsr(shell, tree.root, 0);
- //generate random arguments
- final String[] args = new String[RANDOM.nextInt(NUN_SUBS-1) + 1];
+ final String[] args = new String[NUN_SUBS];
final PermissionStatus[] newstatus = new PermissionStatus[NUN_SUBS];
- final List<Integer> indices = new LinkedList<Integer>();
- for(int i = 0; i < NUN_SUBS; i++) {
- indices.add(i);
- }
- for(int i = 0; i < args.length; i++) {
- final int index = indices.remove(RANDOM.nextInt(indices.size()));
- final String sub = "sub" + index;
- final boolean changeOwner = RANDOM.nextBoolean();
- final boolean changeGroup = RANDOM.nextBoolean();
- final boolean changeMode = !changeOwner && !changeGroup? true:
RANDOM.nextBoolean();
-
- final String owner = changeOwner? sub: "";
- final String group = changeGroup? sub: "";
- final String permission = changeMode? RANDOM.nextInt(8) + "" +
RANDOM.nextInt(8) + "" + RANDOM.nextInt(8): "";
- args[i] = tree.root + "/" + sub + ":" + owner + ":" + group + ":" +
permission;
- newstatus[index] = new ChPermissionStatus(rootstatus, owner, group,
permission);
- }
- for(int i = 0; i < NUN_SUBS; i++) {
- if (newstatus[i] == null) {
- newstatus[i] = new ChPermissionStatus(rootstatus);
- }
- }
+
+ args[0]="/test/testDistCh/sub0:sub1::";
+ newstatus[0] = new ChPermissionStatus(rootstatus, "sub1", "", "");
+
+ args[1]="/test/testDistCh/sub1::sub2:";
+ newstatus[1] = new ChPermissionStatus(rootstatus, "", "sub2", "");
+
+ args[2]="/test/testDistCh/sub2:::437";
+ newstatus[2] = new ChPermissionStatus(rootstatus, "", "", "437");
+
+ args[3]="/test/testDistCh/sub3:sub1:sub2:447";
+ newstatus[3] = new ChPermissionStatus(rootstatus, "sub1", "sub2", "447");
+
+ args[4]="/test/testDistCh/sub4::sub5:437";
+ newstatus[4] = new ChPermissionStatus(rootstatus, "", "sub5", "437");
+
+ args[5]="/test/testDistCh/sub5:sub1:sub5:";
+ newstatus[5] = new ChPermissionStatus(rootstatus, "sub1", "sub5", "");
+
+ args[6]="/test/testDistCh/sub6:sub3::437";
+ newstatus[6] = new ChPermissionStatus(rootstatus, "sub3", "", "437");
+
System.out.println("args=" + Arrays.asList(args).toString().replace(",",
",\n "));
System.out.println("newstatus=" +
Arrays.asList(newstatus).toString().replace(",", ",\n "));
//run DistCh
- new DistCh(mr.createJobConf()).run(args);
+ new DistCh(MiniMRClientClusterFactory.create(this.getClass(), 2,
conf).getConfig()).run(args);
runLsr(shell, tree.root, 0);
//check results
@@ -184,7 +185,7 @@ public class TestDistCh extends junit.fr
}
}
- static final FsPermission UMASK = FsPermission.createImmutable((short)0111);
+
static void checkFileStatus(PermissionStatus expected, FileStatus actual) {
assertEquals(expected.getUserName(), actual.getOwner());
Modified: hadoop/common/branches/branch-trunk-win/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/pom.xml?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/pom.xml Fri Nov 30 19:58:09 2012
@@ -517,7 +517,7 @@ xsi:schemaLocation="http://maven.apache.
<groupId>com.atlassian.maven.plugins</groupId>
<artifactId>maven-clover2-plugin</artifactId>
<configuration>
- <includesAllSourceRoots>true</includesAllSourceRoots>
+ <includesAllSourceRoots>false</includesAllSourceRoots>
<includesTestSourceRoots>true</includesTestSourceRoots>
<licenseLocation>${cloverLicenseLocation}</licenseLocation>
<cloverDatabase>${cloverDatabase}</cloverDatabase>
