Author: mahadev
Date: Mon Aug 18 11:13:17 2008
New Revision: 686840
URL: http://svn.apache.org/viewvc?rev=686840&view=rev
Log:
HADOOP-3962. Shell command fs -count should support paths with different file
systems. (Tsz Wo (Nicholas), SZE via mahadev)
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java
hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Command.java
hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Count.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java
hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDFSShell.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=686840&r1=686839&r2=686840&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Aug 18 11:13:17 2008
@@ -329,6 +329,9 @@
HADOOP-3933. DataNode sometimes sends up to io.byte.per.checksum bytes
more than required to client. (Ning Li via rangadi)
+ HADOOP-3962. Shell command "fs -count" should support paths with different
+ file systems. (Tsz Wo (Nicholas), SZE via mahadev)
+
Release 0.18.0 - 2008-08-19
INCOMPATIBLE CHANGES
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java?rev=686840&r1=686839&r2=686840&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/FsShell.java Mon Aug 18
11:13:17 2008
@@ -1522,7 +1522,7 @@
} else if ("-dus".equals(cmd)) {
dus(argv[i]);
} else if (Count.matches(cmd)) {
- new Count(argv, i, fs).runAll();
+ new Count(argv, i, getConf()).runAll();
} else if ("-ls".equals(cmd)) {
ls(argv[i], false);
} else if ("-lsr".equals(cmd)) {
@@ -1772,7 +1772,7 @@
dus(".");
}
} else if (Count.matches(cmd)) {
- exitCode = new Count(argv, i, fs).runAll();
+ exitCode = new Count(argv, i, getConf()).runAll();
} else if ("-mkdir".equals(cmd)) {
exitCode = doall(cmd, argv, i);
} else if ("-touchz".equals(cmd)) {
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Command.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Command.java?rev=686840&r1=686839&r2=686840&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Command.java
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Command.java Mon Aug
18 11:13:17 2008
@@ -19,6 +19,8 @@
import java.io.*;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -27,13 +29,12 @@
/**
* An abstract class for the execution of a file system command
*/
-abstract public class Command {
- final protected FileSystem fs;
+abstract public class Command extends Configured {
protected String[] args;
/** Constructor */
- protected Command(FileSystem fs) {
- this.fs = fs;
+ protected Command(Configuration conf) {
+ super(conf);
}
/** Return the command's name excluding the leading character - */
@@ -57,6 +58,7 @@
for (String src : args) {
try {
Path srcPath = new Path(src);
+ FileSystem fs = srcPath.getFileSystem(getConf());
FileStatus[] statuses = fs.globStatus(srcPath);
if (statuses == null) {
System.err.println("Can not find listing for " + src);
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Count.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Count.java?rev=686840&r1=686839&r2=686840&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Count.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/shell/Count.java Mon Aug 18
11:13:17 2008
@@ -20,6 +20,7 @@
import java.io.*;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -41,10 +42,9 @@
*
* @param cmd the count command
* @param pos the starting index of the arguments
- * @param fs the file system handler
*/
- public Count(String[] cmd, int pos, FileSystem fs) {
- super(fs);
+ public Count(String[] cmd, int pos, Configuration conf) {
+ super(conf);
CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE, "q");
List<String> parameters = c.parse(cmd, pos);
this.args = parameters.toArray(new String[parameters.size()]);
@@ -70,6 +70,7 @@
@Override
protected void run(Path path) throws IOException {
+ FileSystem fs = path.getFileSystem(getConf());
System.out.println(fs.getContentSummary(path).toString(qOption) + path);
}
}
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java?rev=686840&r1=686839&r2=686840&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java
(original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java Mon
Aug 18 11:13:17 2008
@@ -47,13 +47,15 @@
* An abstract class for the execution of a file system command
*/
abstract private static class DFSAdminCommand extends Command {
+ final DistributedFileSystem dfs;
/** Constructor */
public DFSAdminCommand(FileSystem fs) {
- super(fs);
+ super(fs.getConf());
if (!(fs instanceof DistributedFileSystem)) {
throw new IllegalArgumentException("FileSystem " + fs.getUri() +
" is not a distributed file system");
}
+ this.dfs = (DistributedFileSystem)fs;
}
}
@@ -92,7 +94,7 @@
@Override
public void run(Path path) throws IOException {
- ((DistributedFileSystem)fs).clearQuota(path);
+ dfs.clearQuota(path);
}
}
@@ -139,7 +141,7 @@
@Override
public void run(Path path) throws IOException {
- ((DistributedFileSystem)fs).setQuota(path, quota);
+ dfs.setQuota(path, quota);
}
}
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDFSShell.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDFSShell.java?rev=686840&r1=686839&r2=686840&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDFSShell.java
(original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestDFSShell.java Mon Aug
18 11:13:17 2008
@@ -673,10 +673,20 @@
String root = createTree(dfs, "count");
// Verify the counts
- runCount(root, 2, 4, dfs);
- runCount(root + "2", 2, 1, dfs);
- runCount(root + "2/f1", 0, 1, dfs);
- runCount(root + "2/sub", 1, 0, dfs);
+ runCount(root, 2, 4, conf);
+ runCount(root + "2", 2, 1, conf);
+ runCount(root + "2/f1", 0, 1, conf);
+ runCount(root + "2/sub", 1, 0, conf);
+
+ final FileSystem localfs = FileSystem.getLocal(conf);
+ Path localpath = new Path(TEST_ROOT_DIR, "testcount");
+ localpath = localpath.makeQualified(localfs);
+ localfs.mkdirs(localpath);
+
+ final String localstr = localpath.toString();
+ System.out.println("localstr=" + localstr);
+ runCount(localstr, 1, 0, conf);
+ assertEquals(0, new Count(new String[]{root, localstr}, 0,
conf).runAll());
} finally {
try {
dfs.close();
@@ -685,17 +695,17 @@
cluster.shutdown();
}
}
- private void runCount(String path, long dirs, long files, FileSystem fs
+ private void runCount(String path, long dirs, long files, Configuration conf
) throws IOException {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
PrintStream out = new PrintStream(bytes);
PrintStream oldOut = System.out;
System.setOut(out);
Scanner in = null;
+ String results = null;
try {
- new Count(new String[]{path}, 0, fs).runAll();
- String results = bytes.toString();
- System.out.println(results);
+ new Count(new String[]{path}, 0, conf).runAll();
+ results = bytes.toString();
in = new Scanner(results);
assertEquals(dirs, in.nextLong());
assertEquals(files, in.nextLong());
@@ -703,6 +713,7 @@
if (in!=null) in.close();
IOUtils.closeStream(out);
System.setOut(oldOut);
+ System.out.println("results:\n" + results);
}
}
