Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java Wed Aug 15 19:04:24 2012 @@ -73,6 +73,25 @@ public class DFSck extends Configured im HdfsConfiguration.init(); } + private static final String USAGE = "Usage: DFSck <path> " + + "[-list-corruptfileblocks | " + + "[-move | -delete | -openforwrite] " + + "[-files [-blocks [-locations | -racks]]]]\n" + + "\t<path>\tstart checking from this path\n" + + "\t-move\tmove corrupted files to /lost+found\n" + + "\t-delete\tdelete corrupted files\n" + + "\t-files\tprint out files being checked\n" + + "\t-openforwrite\tprint out files opened for write\n" + + "\t-list-corruptfileblocks\tprint out list of missing " + + "blocks and files they belong to\n" + + "\t-blocks\tprint out block report\n" + + "\t-locations\tprint out locations for every block\n" + + "\t-racks\tprint out network topology for data-node locations\n" + + "\t\tBy default fsck ignores files opened for write, " + + "use -openforwrite to report such files. They are usually " + + " tagged CORRUPT or HEALTHY depending on their block " + + "allocation status"; + private final UserGroupInformation ugi; private final PrintStream out; @@ -93,25 +112,9 @@ public class DFSck extends Configured im /** * Print fsck usage information */ - static void printUsage() { - System.err.println("Usage: DFSck <path> [-list-corruptfileblocks | " + - "[-move | -delete | -openforwrite] " + - "[-files [-blocks [-locations | -racks]]]]"); - System.err.println("\t<path>\tstart checking from this path"); - System.err.println("\t-move\tmove corrupted files to /lost+found"); - System.err.println("\t-delete\tdelete corrupted files"); - System.err.println("\t-files\tprint out files being checked"); - System.err.println("\t-openforwrite\tprint out files opened for write"); - System.err.println("\t-list-corruptfileblocks\tprint out list of missing " - + "blocks and files they belong to"); - System.err.println("\t-blocks\tprint out block report"); - System.err.println("\t-locations\tprint out locations for every block"); - System.err.println("\t-racks\tprint out network topology for data-node locations"); - System.err.println("\t\tBy default fsck ignores files opened for write, " + - "use -openforwrite to report such files. They are usually " + - " tagged CORRUPT or HEALTHY depending on their block " + - "allocation status"); - ToolRunner.printGenericCommandUsage(System.err); + static void printUsage(PrintStream out) { + out.println(USAGE + "\n"); + ToolRunner.printGenericCommandUsage(out); } /** * @param args @@ -119,7 +122,7 @@ public class DFSck extends Configured im @Override public int run(final String[] args) throws IOException { if (args.length == 0) { - printUsage(); + printUsage(System.err); return -1; } @@ -258,12 +261,12 @@ public class DFSck extends Configured im } else { System.err.println("fsck: can only operate on one path at a time '" + args[idx] + "'"); - printUsage(); + printUsage(System.err); return -1; } } else { System.err.println("fsck: Illegal option '" + args[idx] + "'"); - printUsage(); + printUsage(System.err); return -1; } } @@ -304,10 +307,14 @@ public class DFSck extends Configured im // -files option is also used by GenericOptionsParser // Make sure that is not the first argument for fsck int res = -1; - if ((args.length == 0 ) || ("-files".equals(args[0]))) - printUsage(); - else + if ((args.length == 0) || ("-files".equals(args[0]))) { + printUsage(System.err); + ToolRunner.printGenericCommandUsage(System.err); + } else if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) { + res = 0; + } else { res = ToolRunner.run(new DFSck(new HdfsConfiguration()), args); + } System.exit(res); } }
Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java Wed Aug 15 19:04:24 2012 @@ -40,7 +40,6 @@ import org.apache.hadoop.classification. import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.HftpFileSystem; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -48,9 +47,7 @@ import org.apache.hadoop.hdfs.security.t import org.apache.hadoop.hdfs.server.namenode.CancelDelegationTokenServlet; import org.apache.hadoop.hdfs.server.namenode.GetDelegationTokenServlet; import org.apache.hadoop.hdfs.server.namenode.RenewDelegationTokenServlet; -import org.apache.hadoop.hdfs.web.URLUtils; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; @@ -71,8 +68,10 @@ public class DelegationTokenFetcher { private static final String CANCEL = "cancel"; private static final String RENEW = "renew"; private static final String PRINT = "print"; + private static final String HELP = "help"; + private static final String HELP_SHORT = "h"; - private static void printUsage(PrintStream err) throws IOException { + private static void printUsage(PrintStream err) { err.println("fetchdt retrieves delegation tokens from the NameNode"); err.println(); err.println("fetchdt <opts> <token file>"); @@ -107,6 +106,7 @@ public class DelegationTokenFetcher { fetcherOptions.addOption(CANCEL, false, "cancel the token"); fetcherOptions.addOption(RENEW, false, "renew the token"); fetcherOptions.addOption(PRINT, false, "print the token"); + fetcherOptions.addOption(HELP_SHORT, HELP, false, "print out help information"); GenericOptionsParser parser = new GenericOptionsParser(conf, fetcherOptions, args); CommandLine cmd = parser.getCommandLine(); @@ -119,9 +119,14 @@ public class DelegationTokenFetcher { final boolean cancel = cmd.hasOption(CANCEL); final boolean renew = cmd.hasOption(RENEW); final boolean print = cmd.hasOption(PRINT); + final boolean help = cmd.hasOption(HELP); String[] remaining = parser.getRemainingArgs(); // check option validity + if (help) { + printUsage(System.out); + System.exit(0); + } if (cancel && renew || cancel && print || renew && print || cancel && renew && print) { System.err.println("ERROR: Only specify cancel, renew or print."); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java Wed Aug 15 19:04:24 2012 @@ -324,6 +324,10 @@ public class GetConf extends Configured } public static void main(String[] args) throws Exception { + if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) { + System.exit(0); + } + int res = ToolRunner.run(new GetConf(new HdfsConfiguration()), args); System.exit(res); } Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetGroups.java Wed Aug 15 19:04:24 2012 @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.server.namenode.NameNode; @@ -43,6 +44,8 @@ import org.apache.hadoop.util.ToolRunner public class GetGroups extends GetGroupsBase { private static final Log LOG = LogFactory.getLog(GetGroups.class); + + static final String USAGE = "Usage: hdfs groups [username ...]"; static{ HdfsConfiguration.init(); @@ -86,6 +89,10 @@ public class GetGroups extends GetGroups } public static void main(String[] argv) throws Exception { + if (DFSUtil.parseHelpArgument(argv, USAGE, System.out, true)) { + System.exit(0); + } + int res = ToolRunner.run(new GetGroups(new HdfsConfiguration()), argv); System.exit(res); } Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1372630-1373572 Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/test_fuse_dfs.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/test_fuse_dfs.c?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/test_fuse_dfs.c (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/test/test_fuse_dfs.c Wed Aug 15 19:04:24 2012 @@ -75,7 +75,7 @@ static int fuserMount(int *procRet, ...) { int ret, status; size_t i = 0; - char *args[64], *c, *env[] = { NULL }; + char *args[64], *c; va_list ap; pid_t pid, pret; @@ -99,7 +99,7 @@ static int fuserMount(int *procRet, ...) ret, strerror(ret)); return -ret; } else if (pid == 0) { - if (execvpe("fusermount", args, env)) { + if (execvp("fusermount", args)) { ret = errno; fprintf(stderr, "FUSE_TEST: failed to execute fusermount: " "error %d: %s\n", ret, strerror(ret)); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Wed Aug 15 19:04:24 2012 @@ -53,7 +53,7 @@ <name>dfs.datanode.address</name> <value>0.0.0.0:50010</value> <description> - The address where the datanode server will listen to. + The datanode server address and port for data transfer. If the port is 0 then the server will start on a free port. </description> </property> @@ -926,6 +926,22 @@ </property> <property> + <name>dfs.client.use.datanode.hostname</name> + <value>false</value> + <description>Whether clients should use datanode hostnames when + connecting to datanodes. + </description> +</property> + +<property> + <name>dfs.datanode.use.datanode.hostname</name> + <value>false</value> + <description>Whether datanodes should use datanode hostnames when + connecting to other datanodes for data transfer. + </description> +</property> + +<property> <name>dfs.client.local.interfaces</name> <value></value> <description>A comma separated list of network interface names to use Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1372630-1373572 Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1372630-1373572 Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1372630-1373572 Propchange: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1372630-1373572 Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Wed Aug 15 19:04:24 2012 @@ -118,6 +118,8 @@ public class MiniDFSCluster { public static final String PROP_TEST_BUILD_DATA = "test.build.data"; /** Configuration option to set the data dir: {@value} */ public static final String HDFS_MINIDFS_BASEDIR = "hdfs.minidfs.basedir"; + public static final String DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY + = DFS_NAMENODE_SAFEMODE_EXTENSION_KEY + ".testing"; static { DefaultMetricsSystem.setMiniClusterMode(true); } @@ -143,6 +145,7 @@ public class MiniDFSCluster { private boolean setupHostsFile = false; private MiniDFSNNTopology nnTopology = null; private boolean checkExitOnShutdown = true; + private boolean checkDataNodeHostConfig = false; public Builder(Configuration conf) { this.conf = conf; @@ -261,6 +264,14 @@ public class MiniDFSCluster { } /** + * Default: false + */ + public Builder checkDataNodeHostConfig(boolean val) { + this.checkDataNodeHostConfig = val; + return this; + } + + /** * Default: null */ public Builder clusterId(String cid) { @@ -324,7 +335,8 @@ public class MiniDFSCluster { builder.waitSafeMode, builder.setupHostsFile, builder.nnTopology, - builder.checkExitOnShutdown); + builder.checkExitOnShutdown, + builder.checkDataNodeHostConfig); } public class DataNodeProperties { @@ -561,7 +573,7 @@ public class MiniDFSCluster { manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs, operation, racks, hosts, simulatedCapacities, null, true, false, - MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true); + MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false); } private void initMiniDFSCluster( @@ -571,7 +583,8 @@ public class MiniDFSCluster { boolean manageDataDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, String clusterId, boolean waitSafeMode, boolean setupHostsFile, - MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown) + MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown, + boolean checkDataNodeHostConfig) throws IOException { ExitUtil.disableSystemExit(); @@ -587,7 +600,9 @@ public class MiniDFSCluster { int replication = conf.getInt(DFS_REPLICATION_KEY, 3); conf.setInt(DFS_REPLICATION_KEY, Math.min(replication, numDataNodes)); - conf.setInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0); + int safemodeExtension = conf.getInt( + DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY, 0); + conf.setInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, safemodeExtension); conf.setInt(DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 3); // 3 second conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, StaticMapping.class, DNSToSwitchMapping.class); @@ -626,7 +641,7 @@ public class MiniDFSCluster { // Start the DataNodes startDataNodes(conf, numDataNodes, manageDataDfsDirs, operation, racks, - hosts, simulatedCapacities, setupHostsFile); + hosts, simulatedCapacities, setupHostsFile, false, checkDataNodeHostConfig); waitClusterUp(); //make sure ProxyUsers uses the latest conf ProxyUsers.refreshSuperUserGroupsConfiguration(conf); @@ -978,7 +993,21 @@ public class MiniDFSCluster { long[] simulatedCapacities, boolean setupHostsFile) throws IOException { startDataNodes(conf, numDataNodes, manageDfsDirs, operation, racks, hosts, - simulatedCapacities, setupHostsFile, false); + simulatedCapacities, setupHostsFile, false, false); + } + + /** + * @see MiniDFSCluster#startDataNodes(Configuration, int, boolean, StartupOption, + * String[], String[], long[], boolean, boolean, boolean) + */ + public synchronized void startDataNodes(Configuration conf, int numDataNodes, + boolean manageDfsDirs, StartupOption operation, + String[] racks, String[] hosts, + long[] simulatedCapacities, + boolean setupHostsFile, + boolean checkDataNodeAddrConfig) throws IOException { + startDataNodes(conf, numDataNodes, manageDfsDirs, operation, racks, hosts, + simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false); } /** @@ -1004,19 +1033,25 @@ public class MiniDFSCluster { * @param simulatedCapacities array of capacities of the simulated data nodes * @param setupHostsFile add new nodes to dfs hosts files * @param checkDataNodeAddrConfig if true, only set DataNode port addresses if not already set in config + * @param checkDataNodeHostConfig if true, only set DataNode hostname key if not already set in config * * @throws IllegalStateException if NameNode has been shutdown */ public synchronized void startDataNodes(Configuration conf, int numDataNodes, - boolean manageDfsDirs, StartupOption operation, - String[] racks, String[] hosts, - long[] simulatedCapacities, - boolean setupHostsFile, - boolean checkDataNodeAddrConfig) throws IOException { + boolean manageDfsDirs, StartupOption operation, + String[] racks, String[] hosts, + long[] simulatedCapacities, + boolean setupHostsFile, + boolean checkDataNodeAddrConfig, + boolean checkDataNodeHostConfig) throws IOException { if (operation == StartupOption.RECOVER) { return; } - conf.set(DFS_DATANODE_HOST_NAME_KEY, "127.0.0.1"); + if (checkDataNodeHostConfig) { + conf.setIfUnset(DFS_DATANODE_HOST_NAME_KEY, "127.0.0.1"); + } else { + conf.set(DFS_DATANODE_HOST_NAME_KEY, "127.0.0.1"); + } int curDatanodesNum = dataNodes.size(); // for mincluster's the default initialDelay for BRs is 0 Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java Wed Aug 15 19:04:24 2012 @@ -41,6 +41,7 @@ import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Random; import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; @@ -79,8 +80,10 @@ import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.log4j.Level; +import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; import org.mockito.internal.stubbing.answers.ThrowsException; @@ -765,7 +768,7 @@ public class TestDFSClientRetries { try { proxy = DFSUtil.createClientDatanodeProtocolProxy( - fakeDnId, conf, 500, fakeBlock); + fakeDnId, conf, 500, false, fakeBlock); proxy.getReplicaVisibleLength(new ExtendedBlock("bpid", 1)); fail ("Did not get expected exception: SocketTimeoutException"); @@ -842,6 +845,8 @@ public class TestDFSClientRetries { final Path dir = new Path("/testNamenodeRestart"); conf.setBoolean(DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, true); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, 1); + conf.setInt(MiniDFSCluster.DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY, 5000); final short numDatanodes = 3; final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) @@ -864,11 +869,38 @@ public class TestDFSClientRetries { final FileStatus s1 = fs.getFileStatus(file1); assertEquals(length, s1.getLen()); + //create file4, write some data but not close + final Path file4 = new Path(dir, "file4"); + final FSDataOutputStream out4 = fs.create(file4, false, 4096, + fs.getDefaultReplication(file4), 1024L, null); + final byte[] bytes = new byte[1000]; + new Random().nextBytes(bytes); + out4.write(bytes); + out4.write(bytes); + out4.hflush(); + //shutdown namenode assertTrue(HdfsUtils.isHealthy(uri)); cluster.shutdownNameNode(0); assertFalse(HdfsUtils.isHealthy(uri)); + //namenode is down, continue writing file4 in a thread + final Thread file4thread = new Thread(new Runnable() { + @Override + public void run() { + try { + //write some more data and then close the file + out4.write(bytes); + out4.write(bytes); + out4.write(bytes); + out4.close(); + } catch (Exception e) { + exceptions.add(e); + } + } + }); + file4thread.start(); + //namenode is down, read the file in a thread final Thread reader = new Thread(new Runnable() { @Override @@ -927,10 +959,26 @@ public class TestDFSClientRetries { //check file1 and file3 thread.join(); + assertEmpty(exceptions); assertEquals(s1.getLen(), fs.getFileStatus(file3).getLen()); assertEquals(fs.getFileChecksum(file1), fs.getFileChecksum(file3)); reader.join(); + assertEmpty(exceptions); + + //check file4 + file4thread.join(); + assertEmpty(exceptions); + { + final FSDataInputStream in = fs.open(file4); + int count = 0; + for(int r; (r = in.read()) != -1; count++) { + Assert.assertEquals(String.format("count=%d", count), + bytes[count % bytes.length], (byte)r); + } + Assert.assertEquals(5 * bytes.length, count); + in.close(); + } //enter safe mode assertTrue(HdfsUtils.isHealthy(uri)); @@ -970,18 +1018,27 @@ public class TestDFSClientRetries { LOG.info("GOOD!", fnfe); } - if (!exceptions.isEmpty()) { - LOG.error("There are " + exceptions.size() + " exception(s):"); - for(int i = 0; i < exceptions.size(); i++) { - LOG.error("Exception " + i, exceptions.get(i)); - } - fail(); - } + assertEmpty(exceptions); } finally { cluster.shutdown(); } } + static void assertEmpty(final List<Exception> exceptions) { + if (!exceptions.isEmpty()) { + final StringBuilder b = new StringBuilder("There are ") + .append(exceptions.size()) + .append(" exception(s):"); + for(int i = 0; i < exceptions.size(); i++) { + b.append("\n Exception ") + .append(i) + .append(": ") + .append(StringUtils.stringifyException(exceptions.get(i))); + } + fail(b.toString()); + } + } + private static FileSystem createFsWithDifferentUsername( final Configuration conf, final boolean isWebHDFS ) throws IOException, InterruptedException { Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java Wed Aug 15 19:04:24 2012 @@ -417,7 +417,6 @@ public class TestDistributedFileSystem { final Configuration conf = getTestConfiguration(); conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); - conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "localhost"); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem hdfs = cluster.getFileSystem(); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java Wed Aug 15 19:04:24 2012 @@ -171,7 +171,14 @@ public class TestFileCreation { @Test public void testFileCreation() throws IOException { - checkFileCreation(null); + checkFileCreation(null, false); + } + + /** Same test but the client should use DN hostnames */ + @Test + public void testFileCreationUsingHostname() throws IOException { + assumeTrue(System.getProperty("os.name").startsWith("Linux")); + checkFileCreation(null, true); } /** Same test but the client should bind to a local interface */ @@ -180,10 +187,10 @@ public class TestFileCreation { assumeTrue(System.getProperty("os.name").startsWith("Linux")); // The mini cluster listens on the loopback so we can use it here - checkFileCreation("lo"); + checkFileCreation("lo", false); try { - checkFileCreation("bogus-interface"); + checkFileCreation("bogus-interface", false); fail("Able to specify a bogus interface"); } catch (UnknownHostException e) { assertEquals("No such interface bogus-interface", e.getMessage()); @@ -193,16 +200,28 @@ public class TestFileCreation { /** * Test if file creation and disk space consumption works right * @param netIf the local interface, if any, clients should use to access DNs + * @param useDnHostname whether the client should contact DNs by hostname */ - public void checkFileCreation(String netIf) throws IOException { + public void checkFileCreation(String netIf, boolean useDnHostname) + throws IOException { Configuration conf = new HdfsConfiguration(); if (netIf != null) { conf.set(DFSConfigKeys.DFS_CLIENT_LOCAL_INTERFACES, netIf); } + conf.setBoolean(DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME, useDnHostname); + if (useDnHostname) { + // Since the mini cluster only listens on the loopback we have to + // ensure the hostname used to access DNs maps to the loopback. We + // do this by telling the DN to advertise localhost as its hostname + // instead of the default hostname. + conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "localhost"); + } if (simulatedStorage) { SimulatedFSDataset.setFactory(conf); } - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .checkDataNodeHostConfig(true) + .build(); FileSystem fs = cluster.getFileSystem(); try { Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java Wed Aug 15 19:04:24 2012 @@ -92,7 +92,6 @@ public class TestHftpFileSystem { RAN.setSeed(seed); config = new Configuration(); - config.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "localhost"); cluster = new MiniDFSCluster.Builder(config).numDataNodes(2).build(); hdfs = cluster.getFileSystem(); blockPoolId = cluster.getNamesystem().getBlockPoolId(); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java Wed Aug 15 19:04:24 2012 @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; import java.io.File; @@ -41,6 +42,7 @@ public class TestMiniDFSCluster { private static final String CLUSTER_2 = "cluster2"; private static final String CLUSTER_3 = "cluster3"; private static final String CLUSTER_4 = "cluster4"; + private static final String CLUSTER_5 = "cluster5"; protected String testDataPath; protected File testDataDir; @Before @@ -125,4 +127,25 @@ public class TestMiniDFSCluster { } } } + + /** MiniDFSCluster should not clobber dfs.datanode.hostname if requested */ + @Test(timeout=100000) + public void testClusterSetDatanodeHostname() throws Throwable { + assumeTrue(System.getProperty("os.name").startsWith("Linux")); + Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "MYHOST"); + File testDataCluster5 = new File(testDataPath, CLUSTER_5); + String c5Path = testDataCluster5.getAbsolutePath(); + conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c5Path); + MiniDFSCluster cluster5 = new MiniDFSCluster.Builder(conf) + .numDataNodes(1) + .checkDataNodeHostConfig(true) + .build(); + try { + assertEquals("DataNode hostname config not respected", "MYHOST", + cluster5.getDataNodes().get(0).getDatanodeId().getHostName()); + } finally { + MiniDFSCluster.shutdownCluster(cluster5); + } + } } Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java Wed Aug 15 19:04:24 2012 @@ -246,7 +246,7 @@ public class TestShortCircuitLocalRead { @Override public ClientDatanodeProtocol run() throws Exception { return DFSUtil.createClientDatanodeProtocolProxy(dnInfo, conf, - 60000); + 60000, false); } }); @@ -264,7 +264,7 @@ public class TestShortCircuitLocalRead { @Override public ClientDatanodeProtocol run() throws Exception { return DFSUtil.createClientDatanodeProtocolProxy(dnInfo, conf, - 60000); + 60000, false); } }); try { Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java Wed Aug 15 19:04:24 2012 @@ -304,7 +304,7 @@ public class TestBlockToken { long endTime = Time.now() + 3000; while (Time.now() < endTime) { proxy = DFSUtil.createClientDatanodeProtocolProxy(fakeDnId, conf, 1000, - fakeBlock); + false, fakeBlock); assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3)); if (proxy != null) { RPC.stopProxy(proxy); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java Wed Aug 15 19:04:24 2012 @@ -105,10 +105,13 @@ public class DataNodeTestUtils { } public static InterDatanodeProtocol createInterDatanodeProtocolProxy( - DataNode dn, DatanodeID datanodeid, final Configuration conf - ) throws IOException { + DataNode dn, DatanodeID datanodeid, final Configuration conf, + boolean connectToDnViaHostname) throws IOException { + if (connectToDnViaHostname != dn.getDnConf().connectToDnViaHostname) { + throw new AssertionError("Unexpected DN hostname configuration"); + } return DataNode.createInterDataNodeProtocolProxy(datanodeid, conf, - dn.getDnConf().socketTimeout); + dn.getDnConf().socketTimeout, dn.getDnConf().connectToDnViaHostname); } public static void shutdownBlockScanner(DataNode dn) { Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java Wed Aug 15 19:04:24 2012 @@ -29,6 +29,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSClientAdapter; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -59,6 +60,8 @@ import org.apache.hadoop.net.NetUtils; import org.junit.Assert; import org.junit.Test; +import static org.junit.Assume.assumeTrue; + /** * This tests InterDataNodeProtocol for block handling. */ @@ -125,17 +128,42 @@ public class TestInterDatanodeProtocol { return blocks.get(blocks.size() - 1); } + /** Test block MD access via a DN */ + @Test + public void testBlockMetaDataInfo() throws Exception { + checkBlockMetaDataInfo(false); + } + + /** The same as above, but use hostnames for DN<->DN communication */ + @Test + public void testBlockMetaDataInfoWithHostname() throws Exception { + assumeTrue(System.getProperty("os.name").startsWith("Linux")); + checkBlockMetaDataInfo(true); + } + /** * The following test first creates a file. * It verifies the block information from a datanode. - * Then, it updates the block with new information and verifies again. + * Then, it updates the block with new information and verifies again. + * @param useDnHostname whether DNs should connect to other DNs by hostname */ - @Test - public void testBlockMetaDataInfo() throws Exception { + private void checkBlockMetaDataInfo(boolean useDnHostname) throws Exception { MiniDFSCluster cluster = null; + conf.setBoolean(DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, useDnHostname); + if (useDnHostname) { + // Since the mini cluster only listens on the loopback we have to + // ensure the hostname used to access DNs maps to the loopback. We + // do this by telling the DN to advertise localhost as its hostname + // instead of the default hostname. + conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "localhost"); + } + try { - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(3) + .checkDataNodeHostConfig(true) + .build(); cluster.waitActive(); //create a file @@ -154,7 +182,7 @@ public class TestInterDatanodeProtocol { //connect to a data node DataNode datanode = cluster.getDataNode(datanodeinfo[0].getIpcPort()); InterDatanodeProtocol idp = DataNodeTestUtils.createInterDatanodeProtocolProxy( - datanode, datanodeinfo[0], conf); + datanode, datanodeinfo[0], conf, useDnHostname); //stop block scanner, so we could compare lastScanTime DataNodeTestUtils.shutdownBlockScanner(datanode); @@ -364,7 +392,7 @@ public class TestInterDatanodeProtocol { try { proxy = DataNode.createInterDataNodeProtocolProxy( - dInfo, conf, 500); + dInfo, conf, 500, false); proxy.initReplicaRecovery(new RecoveringBlock( new ExtendedBlock("bpid", 1), null, 100)); fail ("Expected SocketTimeoutException exception, but did not get."); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Wed Aug 15 19:04:24 2012 @@ -119,6 +119,11 @@ public class TestEditLog { "a4ff 0000 0000 0000 0000 0000 0000 0000" ).replace(" ","")); + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } static final byte TRAILER_BYTE = FSEditLogOpCodes.OP_INVALID.getOpCode(); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java Wed Aug 15 19:04:24 2012 @@ -40,6 +40,12 @@ public class TestEditLogFileOutputStream final static int MIN_PREALLOCATION_LENGTH = EditLogFileOutputStream.MIN_PREALLOCATION_LENGTH; + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } + @Before @After public void deleteEditsFile() { Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java Wed Aug 15 19:04:24 2012 @@ -51,6 +51,12 @@ import com.google.common.collect.Immutab public class TestFileJournalManager { static final Log LOG = LogFactory.getLog(TestFileJournalManager.class); + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } + /** * Find out how many transactions we can read from a * FileJournalManager, starting at a given transaction ID. Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java Wed Aug 15 19:04:24 2012 @@ -57,6 +57,7 @@ public class TestNameNodeRecovery { static { recoverStartOpt.setForce(MetaRecoveryContext.FORCE_ALL); + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); } static void runEditLogTest(EditLogTestSetup elts) throws IOException { Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java Wed Aug 15 19:04:24 2012 @@ -49,6 +49,12 @@ public class TestSecurityTokenEditLog { static final int NUM_THREADS = 100; static final int opsPerTrans = 3; + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } + // // an object that does a bunch of transactions // Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java Wed Aug 15 19:04:24 2012 @@ -65,7 +65,7 @@ public abstract class HATestUtil { * @throws CouldNotCatchUpException if the standby doesn't catch up to the * active in NN_LAG_TIMEOUT milliseconds */ - static void waitForStandbyToCatchUp(NameNode active, + public static void waitForStandbyToCatchUp(NameNode active, NameNode standby) throws InterruptedException, IOException, CouldNotCatchUpException { long activeTxId = active.getNamesystem().getFSImage().getEditLog() Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java Wed Aug 15 19:04:24 2012 @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.permission.F import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.hdfs.server.namenode.NNStorage; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; @@ -52,6 +53,12 @@ public class TestEditLogsDuringFailover private static final Log LOG = LogFactory.getLog(TestEditLogsDuringFailover.class); private static final int NUM_DIRS_IN_LOG = 5; + + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } @Test public void testStartup() throws Exception { Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java Wed Aug 15 19:04:24 2012 @@ -159,6 +159,13 @@ public class TestInitializeSharedEdits { } @Test + public void testFailWhenNoSharedEditsSpecified() throws Exception { + Configuration confNoShared = new Configuration(conf); + confNoShared.unset(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY); + assertFalse(NameNode.initializeSharedEdits(confNoShared, true)); + } + + @Test public void testDontOverWriteExistingDir() { assertFalse(NameNode.initializeSharedEdits(conf, false)); assertTrue(NameNode.initializeSharedEdits(conf, false)); Modified: hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java?rev=1373573&r1=1373572&r2=1373573&view=diff ============================================================================== --- hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java (original) +++ hadoop/common/branches/HDFS-3077/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java Wed Aug 15 19:04:24 2012 @@ -55,7 +55,9 @@ public class TestDFSHAAdmin { private DFSHAAdmin tool; private ByteArrayOutputStream errOutBytes = new ByteArrayOutputStream(); + private ByteArrayOutputStream outBytes = new ByteArrayOutputStream(); private String errOutput; + private String output; private HAServiceProtocol mockProtocol; private ZKFCProtocol mockZkfcProtocol; @@ -111,12 +113,14 @@ public class TestDFSHAAdmin { }; tool.setConf(getHAConf()); tool.setErrOut(new PrintStream(errOutBytes)); + tool.setOut(new PrintStream(outBytes)); } private void assertOutputContains(String string) { - if (!errOutput.contains(string)) { - fail("Expected output to contain '" + string + "' but was:\n" + - errOutput); + if (!errOutput.contains(string) && !output.contains(string)) { + fail("Expected output to contain '" + string + + "' but err_output was:\n" + errOutput + + "\n and output was: \n" + output); } } @@ -143,7 +147,7 @@ public class TestDFSHAAdmin { @Test public void testHelp() throws Exception { - assertEquals(-1, runTool("-help")); + assertEquals(0, runTool("-help")); assertEquals(0, runTool("-help", "transitionToActive")); assertOutputContains("Transitions the service into Active"); } @@ -378,10 +382,12 @@ public class TestDFSHAAdmin { private Object runTool(String ... args) throws Exception { errOutBytes.reset(); + outBytes.reset(); LOG.info("Running: DFSHAAdmin " + Joiner.on(" ").join(args)); int ret = tool.run(args); errOutput = new String(errOutBytes.toByteArray(), Charsets.UTF_8); - LOG.info("Output:\n" + errOutput); + output = new String(outBytes.toByteArray(), Charsets.UTF_8); + LOG.info("Err_output:\n" + errOutput + "\nOutput:\n" + output); return ret; }
