Author: szetszwo
Date: Tue Mar 10 23:01:39 2009
New Revision: 752292
URL: http://svn.apache.org/viewvc?rev=752292&view=rev
Log:
HADOOP-5258. Add a new DFSAdmin command to print a tree of the rack and
datanode topology as seen by the namenode. (Jakob Homan via szetszwo)
Added:
hadoop/core/trunk/src/test/org/apache/hadoop/cli/util/RegexpAcrossOutputComparator.java
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/core/org/apache/hadoop/net/NetUtils.java
hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java
hadoop/core/trunk/src/docs/src/documentation/content/xdocs/commands_manual.xml
hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java
hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java
hadoop/core/trunk/src/test/org/apache/hadoop/cli/testConf.xml
hadoop/core/trunk/src/test/org/apache/hadoop/cli/util/ComparatorData.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=752292&r1=752291&r2=752292&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Mar 10 23:01:39 2009
@@ -52,6 +52,9 @@
HADOOP-5144. Add a new DFSAdmin command for changing the setting of restore
failed storage replicas in namenode. (Boris Shkolnik via szetszwo)
+ HADOOP-5258. Add a new DFSAdmin command to print a tree of the rack and
+ datanode topology as seen by the namenode. (Jakob Homan via szetszwo)
+
IMPROVEMENTS
HADOOP-4565. Added CombineFileInputFormat to use data locality information
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/net/NetUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/net/NetUtils.java?rev=752292&r1=752291&r2=752292&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/net/NetUtils.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/net/NetUtils.java Tue Mar 10
23:01:39 2009
@@ -28,6 +28,7 @@
import java.net.UnknownHostException;
import java.nio.channels.SocketChannel;
import java.util.Map.Entry;
+import java.util.regex.Pattern;
import java.util.*;
import javax.net.SocketFactory;
@@ -441,4 +442,39 @@
}
return hostNames;
}
+
+ /**
+ * Attempt to obtain the host name of a name specified by ip address.
+ * Check that the node name is an ip addr and if so, attempt to determine
+ * its host name. If the name is not an IP addr, or the actual name cannot
+ * be determined, return null.
+ *
+ * @return Host name or null
+ */
+ private static final Pattern ipPattern = // Pattern for matching hostname to
ip:port
+ Pattern.compile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:?\\d*");
+ public static String getHostNameOfIP(String ip) {
+ // If name is not an ip addr, don't bother looking it up
+ if(!ipPattern.matcher(ip).matches())
+ return null;
+
+ String hostname = "";
+ try {
+ String n = ip.substring(0, ip.indexOf(':'));
+ hostname = InetAddress.getByName(n).getHostName();
+ } catch (UnknownHostException e) {
+ return null;
+ }
+
+ return hostname;
+ }
+
+ /**
+ * Return hostname without throwing exception.
+ * @return hostname
+ */
+ public static String getHostname() {
+ try {return "" + InetAddress.getLocalHost();}
+ catch(UnknownHostException uhe) {return "" + uhe;}
+ }
}
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java?rev=752292&r1=752291&r2=752292&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/StringUtils.java Tue Mar
10 23:01:39 2009
@@ -20,22 +20,21 @@
import java.io.PrintWriter;
import java.io.StringWriter;
-import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
-import java.net.UnknownHostException;
import java.text.DateFormat;
import java.text.DecimalFormat;
import java.text.NumberFormat;
-import java.util.Locale;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Date;
import java.util.List;
+import java.util.Locale;
import java.util.StringTokenizer;
-import java.util.Collection;
-import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.net.NetUtils;
/**
* General string utils
@@ -504,15 +503,6 @@
}
/**
- * Return hostname without throwing exception.
- * @return hostname
- */
- public static String getHostname() {
- try {return "" + InetAddress.getLocalHost();}
- catch(UnknownHostException uhe) {return "" + uhe;}
- }
-
- /**
* Return a message for logging.
* @param prefix prefix keyword for the message
* @param msg content of the message
@@ -535,7 +525,7 @@
*/
public static void startupShutdownMessage(Class<?> clazz, String[] args,
final org.apache.commons.logging.Log LOG)
{
- final String hostname = getHostname();
+ final String hostname = NetUtils.getHostname();
final String classname = clazz.getSimpleName();
LOG.info(
toStartupShutdownString("STARTUP_MSG: ", new String[] {
Modified:
hadoop/core/trunk/src/docs/src/documentation/content/xdocs/commands_manual.xml
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/docs/src/documentation/content/xdocs/commands_manual.xml?rev=752292&r1=752291&r2=752292&view=diff
==============================================================================
---
hadoop/core/trunk/src/docs/src/documentation/content/xdocs/commands_manual.xml
(original)
+++
hadoop/core/trunk/src/docs/src/documentation/content/xdocs/commands_manual.xml
Tue Mar 10 23:01:39 2009
@@ -516,6 +516,11 @@
This completes the upgrade process.</td>
</tr>
<tr>
+ <td><code>-printTopology</code></td>
+ <td>Print a tree of the rack/datanode
topology of the
+ cluster as seen by the NameNode.</td>
+ </tr>
+ <tr>
<td><code>-upgradeProgress status |
details | force</code></td>
<td>Request current distributed upgrade
status,
a detailed status or force the upgrade to proceed.</td>
Modified:
hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml?rev=752292&r1=752291&r2=752292&view=diff
==============================================================================
---
hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml
(original)
+++
hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml
Tue Mar 10 23:01:39 2009
@@ -200,6 +200,11 @@
been marked for decommission. Entires not present in both the lists
are decommissioned.
</li>
+ <li>
+ <code>-printTopology</code>
+ : Print the topology of the cluster. Display a tree of racks and
+ datanodes attached to the tracks as viewed by the NameNode.
+ </li>
</ul>
<p>
For command usage, see <a
href="commands_manual.html#dfsadmin">dfsadmin command</a>.
Modified:
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java?rev=752292&r1=752291&r2=752292&view=diff
==============================================================================
---
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
(original)
+++
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
Tue Mar 10 23:01:39 2009
@@ -20,16 +20,14 @@
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
import java.util.Date;
-import java.util.regex.Pattern;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.net.NodeBase;
@@ -47,10 +45,8 @@
protected long lastUpdate;
protected int xceiverCount;
protected String location = NetworkTopology.DEFAULT_RACK;
- static final Pattern ip = // Pattern for matching hostname to ip:port
- Pattern.compile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:?\\d*");
- /** HostName as suplied by the datanode during registration as its
+ /** HostName as supplied by the datanode during registration as its
* name. Namenode uses datanode IP address as the name.
*/
protected String hostName = null;
@@ -152,7 +148,7 @@
this.xceiverCount = xceiverCount;
}
- /** rack name **/
+ /** rack name */
public synchronized String getNetworkLocation() {return location;}
/** Sets the rack name */
@@ -177,7 +173,7 @@
long nonDFSUsed = getNonDfsUsed();
float usedPercent = getDfsUsedPercent();
float remainingPercent = getRemainingPercent();
- String hostName = getHostNameOfIP();
+ String hostName = NetUtils.getHostNameOfIP(name);
buffer.append("Name: "+ name);
if(hostName != null)
@@ -205,30 +201,6 @@
return buffer.toString();
}
- /**
- * Attempt to obtain the host name of a name specified by ip address.
- * Check that the node name is an ip addr and if so, attempt to determine
- * its host name. If the name is not an IP addr, or the actual name cannot
- * be determined, return null.
- *
- * @return Host name or null
- */
- private String getHostNameOfIP() {
- // If name is not an ip addr, don't bother looking it up
- if(!ip.matcher(name).matches())
- return null;
-
- String hostname = "";
- try {
- String n = name.substring(0, name.indexOf(':'));
- hostname = InetAddress.getByName(n).getHostName();
- } catch (UnknownHostException e) {
- return null;
- }
-
- return hostname;
- }
-
/** A formatted string for printing the status of the DataNode. */
public String dumpDatanode() {
StringBuffer buffer = new StringBuffer();
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java?rev=752292&r1=752291&r2=752292&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java
(original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java Tue
Mar 10 23:01:39 2009
@@ -18,7 +18,11 @@
package org.apache.hadoop.hdfs.tools;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.TreeSet;
import javax.security.auth.login.LoginException;
@@ -29,6 +33,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.Command;
import org.apache.hadoop.fs.shell.CommandFormat;
+import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.FSConstants;
@@ -463,6 +468,7 @@
"\t[" + SetSpaceQuotaCommand.USAGE + "]\n" +
"\t[" + ClearSpaceQuotaCommand.USAGE +"]\n" +
"\t[-refreshServiceAcl]\n" +
+ "\t[-printTopology]\n" +
"\t[-help [cmd]]\n";
String report ="-report: \tReports basic filesystem information and
statistics.\n";
@@ -516,6 +522,9 @@
String refreshServiceAcl = "-refreshServiceAcl: Reload the service-level
authorization policy file\n" +
"\t\tNamenode will reload the authorization policy file.\n";
+ String printTopology = "-printTopology: Print a tree of the racks and
their\n" +
+ "\t\tnodes as reported by the Namenode\n";
+
String help = "-help [cmd]: \tDisplays help for the given command or all
commands if none\n" +
"\t\tis specified.\n";
@@ -545,6 +554,8 @@
System.out.println(ClearSpaceQuotaCommand.DESCRIPTION);
} else if ("refreshServiceAcl".equals(cmd)) {
System.out.println(refreshServiceAcl);
+ } else if ("printTopology".equals(cmd)) {
+ System.out.println(printTopology);
} else if ("help".equals(cmd)) {
System.out.println(help);
} else {
@@ -562,6 +573,7 @@
System.out.println(SetSpaceQuotaCommand.DESCRIPTION);
System.out.println(ClearSpaceQuotaCommand.DESCRIPTION);
System.out.println(refreshServiceAcl);
+ System.out.println(printTopology);
System.out.println(help);
System.out.println();
ToolRunner.printGenericCommandUsage(System.out);
@@ -569,7 +581,6 @@
}
-
/**
* Command to ask the namenode to finalize previously performed upgrade.
* Usage: java DFSAdmin -finalizeUpgrade
@@ -645,6 +656,54 @@
return 0;
}
+ /**
+ * Display each rack and the nodes assigned to that rack, as determined
+ * by the NameNode, in a hierarchical manner. The nodes and racks are
+ * sorted alphabetically.
+ *
+ * @throws IOException If an error while getting datanode report
+ */
+ public int printTopology() throws IOException {
+ if (fs instanceof DistributedFileSystem) {
+ DistributedFileSystem dfs = (DistributedFileSystem)fs;
+ DFSClient client = dfs.getClient();
+ DatanodeInfo[] report = client.datanodeReport(DatanodeReportType.ALL);
+
+ // Build a map of rack -> nodes from the datanode report
+ HashMap<String, TreeSet<String> > tree = new HashMap<String,
TreeSet<String>>();
+ for(DatanodeInfo dni : report) {
+ String location = dni.getNetworkLocation();
+ String name = dni.getName();
+
+ if(!tree.containsKey(location)) {
+ tree.put(location, new TreeSet<String>());
+ }
+
+ tree.get(location).add(name);
+ }
+
+ // Sort the racks (and nodes) alphabetically, display in order
+ ArrayList<String> racks = new ArrayList<String>(tree.keySet());
+ Collections.sort(racks);
+
+ for(String r : racks) {
+ System.out.println("Rack: " + r);
+ TreeSet<String> nodes = tree.get(r);
+
+ for(String n : nodes) {
+ System.out.print(" " + n);
+ String hostname = NetUtils.getHostNameOfIP(n);
+ if(hostname != null)
+ System.out.print(" (" + hostname + ")");
+ System.out.println();
+ }
+
+ System.out.println();
+ }
+ }
+ return 0;
+ }
+
private static UnixUserGroupInformation getUGI(Configuration conf)
throws IOException {
UnixUserGroupInformation ugi = null;
@@ -725,6 +784,9 @@
} else if ("-refreshServiceAcl".equals(cmd)) {
System.err.println("Usage: java DFSAdmin"
+ " [-refreshServiceAcl]");
+ } else if ("-printTopology".equals(cmd)) {
+ System.err.println("Usage: java DFSAdmin"
+ + " [-printTopology]");
} else {
System.err.println("Usage: java DFSAdmin");
System.err.println(" [-report]");
@@ -736,6 +798,7 @@
System.err.println(" [-upgradeProgress status | details |
force]");
System.err.println(" [-metasave filename]");
System.err.println(" [-refreshServiceAcl]");
+ System.err.println(" [-printTopology]");
System.err.println(" ["+SetQuotaCommand.USAGE+"]");
System.err.println(" ["+ClearQuotaCommand.USAGE+"]");
System.err.println(" ["+SetSpaceQuotaCommand.USAGE+"]");
@@ -811,6 +874,12 @@
printUsage(cmd);
return exitCode;
}
+ else if ("-printTopology".equals(cmd)) {
+ if(argv.length != 1) {
+ printUsage(cmd);
+ return exitCode;
+ }
+ }
}
// initialize DFSAdmin
@@ -853,6 +922,8 @@
exitCode = new SetSpaceQuotaCommand(argv, i, fs).runAll();
} else if ("-refreshServiceAcl".equals(cmd)) {
exitCode = refreshServiceAcl();
+ } else if ("-printTopology".equals(cmd)) {
+ exitCode = printTopology();
} else if ("-help".equals(cmd)) {
if (i < argv.length) {
printHelp(argv[i]);
@@ -871,7 +942,7 @@
} catch (RemoteException e) {
//
// This is a error returned by hadoop server. Print
- // out the first line of the error mesage, ignore the stack trace.
+ // out the first line of the error message, ignore the stack trace.
exitCode = -1;
try {
String[] content;
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java?rev=752292&r1=752291&r2=752292&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/cli/TestCLI.java Tue Mar 10
23:01:39 2009
@@ -121,7 +121,17 @@
conf.setBoolean(ServiceAuthorizationManager.SERVICE_AUTHORIZATION_CONFIG,
true);
- dfsCluster = new MiniDFSCluster(conf, 1, true, null);
+ // Many of the tests expect a replication value of 1 in the output
+ conf.setInt("dfs.replication", 1);
+
+ // Build racks and hosts configuration to test dfsAdmin -printTopology
+ String [] racks = {"/rack1", "/rack1", "/rack2", "/rack2",
+ "/rack2", "/rack3", "/rack4", "/rack4" };
+ String [] hosts = {"host1", "host2", "host3", "host4",
+ "host5", "host6", "host7", "host8" };
+
+ dfsCluster = new MiniDFSCluster(conf, 8, true, racks, hosts);
+
namenode = conf.get("fs.default.name", "file:///");
clitestDataDir = new File(TEST_CACHE_DATA_DIR).
toURI().toString().replace(' ', '+');
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/cli/testConf.xml
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/cli/testConf.xml?rev=752292&r1=752291&r2=752292&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/cli/testConf.xml (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/cli/testConf.xml Tue Mar 10
23:01:39 2009
@@ -3335,5 +3335,33 @@
</comparators>
</test>
+ <test> <!-- Tested -->
+ <description>printTopology: verifying that the topology map is what we
expect</description>
+ <test-commands>
+ <dfs-admin-command>-fs NAMENODE -printTopology</dfs-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <!-- No cleanup -->
+ </cleanup-commands>
+ <comparators>
+ <!-- miniDFS cluster started in TestCLI is set to match this output -->
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>^Rack:
\/rack1\s*127.0.0.1:\d+\s\(localhost\)\s*127.0.0.1:\d+\s\(localhost\)</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>Rack:
\/rack2\s*127.0.0.1:\d+\s\(localhost\)\s*127.0.0.1:\d+\s\(localhost\)\s*127.0.0.1:\d+\s\(localhost\)</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>Rack:
\/rack3\s*127.0.0.1:\d+\s\(localhost\)</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>Rack:
\/rack4\s*127.0.0.1:\d+\s\(localhost\)\s*127.0.0.1:\d+\s\(localhost\)</expected-output>
+ </comparator>
+ </comparators>
+ </test>
</tests>
</configuration>
Modified:
hadoop/core/trunk/src/test/org/apache/hadoop/cli/util/ComparatorData.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/cli/util/ComparatorData.java?rev=752292&r1=752291&r2=752292&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/cli/util/ComparatorData.java
(original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/cli/util/ComparatorData.java
Tue Mar 10 23:01:39 2009
@@ -18,8 +18,6 @@
package org.apache.hadoop.cli.util;
-import java.util.Vector;
-
/**
*
* Class to store CLI Test Comparators Data
Added:
hadoop/core/trunk/src/test/org/apache/hadoop/cli/util/RegexpAcrossOutputComparator.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/cli/util/RegexpAcrossOutputComparator.java?rev=752292&view=auto
==============================================================================
---
hadoop/core/trunk/src/test/org/apache/hadoop/cli/util/RegexpAcrossOutputComparator.java
(added)
+++
hadoop/core/trunk/src/test/org/apache/hadoop/cli/util/RegexpAcrossOutputComparator.java
Tue Mar 10 23:01:39 2009
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cli.util;
+
+import java.util.regex.Pattern;
+
+/**
+ * Comparator for command line tests that attempts to find a regexp
+ * within the entire text returned by a command.
+ *
+ * This comparator differs from RegexpComparator in that it attempts
+ * to match the pattern within all of the text returned by the command,
+ * rather than matching against each line of the returned text. This
+ * allows matching against patterns that span multiple lines.
+ */
+public class RegexpAcrossOutputComparator extends ComparatorBase {
+
+ @Override
+ public boolean compare(String actual, String expected) {
+ return Pattern.compile(expected).matcher(actual).find();
+ }
+
+}
