svn commit: r674627 - in /hadoop/core/trunk: CHANGES.txt src/test/org/apache/hadoop/mapred/SortValidator.java

[cdouglas]

Author: cdouglas
Date: Mon Jul  7 13:44:32 2008
New Revision: 674627

URL: http://svn.apache.org/viewvc?rev=674627&view=rev
Log:
HADOOP-3597. Fix SortValidator to use filesystems other than the default as
input. Validation job still runs on default fs. Contributed by Jothi
Padmanabhan.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/SortValidator.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=674627&r1=674626&r2=674627&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Jul  7 13:44:32 2008
@@ -764,6 +764,10 @@
     HADOOP-3683. Fix dfs metrics to count file listings rather than files
     listed. (lohit vijayarenu via cdouglas)
 
+    HADOOP-3597. Fix SortValidator to use filesystems other than the default as
+    input. Validation job still runs on default fs.
+    (Jothi Padmanabhan via cdouglas)
+
 Release 0.17.1 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/SortValidator.java
URL: 
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/SortValidator.java?rev=674627&r1=674626&r2=674627&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/SortValidator.java 
(original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/SortValidator.java Mon 
Jul  7 13:44:32 2008
@@ -314,14 +314,16 @@
     
     static void checkRecords(Configuration defaults, 
                              Path sortInput, Path sortOutput) throws 
IOException {
-      FileSystem fs = FileSystem.get(defaults);
+      FileSystem inputfs = sortInput.getFileSystem(defaults);
+      FileSystem outputfs = sortOutput.getFileSystem(defaults);
+      FileSystem defaultfs = FileSystem.get(defaults);
       JobConf jobConf = new JobConf(defaults, RecordStatsChecker.class);
       jobConf.setJobName("sortvalidate-recordstats-checker");
 
       int noSortReduceTasks = 
-        fs.listStatus(sortOutput, sortPathsFilter).length;
+        outputfs.listStatus(sortOutput, sortPathsFilter).length;
       jobConf.setInt("sortvalidate.sort.reduce.tasks", noSortReduceTasks);
-      int noSortInputpaths = fs.listStatus(sortInput).length;
+      int noSortInputpaths =  inputfs.listStatus(sortInput).length;
 
       jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class);
       jobConf.setOutputFormat(SequenceFileOutputFormat.class);
@@ -339,8 +341,8 @@
       FileInputFormat.setInputPaths(jobConf, sortInput);
       FileInputFormat.addInputPath(jobConf, sortOutput);
       Path outputPath = new Path("/tmp/sortvalidate/recordstatschecker");
-      if (fs.exists(outputPath)) {
-        fs.delete(outputPath, true);
+      if (defaultfs.exists(outputPath)) {
+        defaultfs.delete(outputPath, true);
       }
       FileOutputFormat.setOutputPath(jobConf, outputPath);
       
@@ -365,7 +367,7 @@
       
       // Check to ensure that the statistics of the 
       // framework's sort-input and sort-output match
-      SequenceFile.Reader stats = new SequenceFile.Reader(fs, 
+      SequenceFile.Reader stats = new SequenceFile.Reader(defaultfs,
                                                           new Path(outputPath, 
"part-00000"), defaults);
       IntWritable k1 = new IntWritable();
       IntWritable k2 = new IntWritable();