svn commit: r657444 - in /lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop: RecommenderJob.java RecommenderMapper.java

[srowen]

Author: srowen
Date: Sat May 17 12:47:50 2008
New Revision: 657444

URL: http://svn.apache.org/viewvc?rev=657444&view=rev
Log:
Try to use FileSystem abstraction correctly to make this work remotely

Modified:
    
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderJob.java
    
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderMapper.java

Modified: 
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderJob.java
URL: 
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderJob.java?rev=657444&r1=657443&r2=657444&view=diff
==============================================================================
--- 
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderJob.java
 (original)
+++ 
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderJob.java
 Sat May 17 12:47:50 2008
@@ -17,6 +17,7 @@
 
 package org.apache.mahout.cf.taste.hadoop;
 
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.JobClient;
@@ -70,9 +71,19 @@
                                      String userIDFile,
                                      String dataModelFile,
                                      String outputPath,
-                                     int numMappers) {
+                                     int numMappers) throws IOException {
+
+    Path userIDFilePath = new Path(userIDFile);
+    Path outputPathPath = new Path(outputPath);
+
     JobConf jobConf = new JobConf(Recommender.class);
 
+    FileSystem fs = FileSystem.get(jobConf);
+    if (fs.exists(outputPathPath)) {
+      fs.delete(outputPathPath);
+    }
+    fs.mkdirs(outputPathPath);
+
     jobConf.set(RecommenderMapper.RECOMMENDER_CLASS_NAME, recommendClassName);
     jobConf.set(RecommenderMapper.RECOMMENDATIONS_PER_USER, 
String.valueOf(recommendationsPerUser));
     jobConf.set(RecommenderMapper.DATA_MODEL_FILE, dataModelFile);
@@ -80,7 +91,7 @@
     jobConf.setJobName(RecommenderJob.class.getSimpleName());
 
     jobConf.setInputFormat(TextInputFormat.class);
-    jobConf.setInputPath(new Path(userIDFile));
+    jobConf.setInputPath(userIDFilePath);
 
     jobConf.setNumMapTasks(numMappers);
     jobConf.setMapperClass(RecommenderMapper.class);
@@ -93,7 +104,8 @@
     jobConf.setOutputValueClass(RecommendedItemsWritable.class);
 
     jobConf.setOutputFormat(TextOutputFormat.class);
-    jobConf.setOutputPath(new Path(outputPath));
+    jobConf.setOutputPath(outputPathPath);
+
     return jobConf;
   }
 

Modified: 
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderMapper.java
URL: 
http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderMapper.java?rev=657444&r1=657443&r2=657444&view=diff
==============================================================================
--- 
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderMapper.java
 (original)
+++ 
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderMapper.java
 Sat May 17 12:47:50 2008
@@ -17,6 +17,8 @@
 
 package org.apache.mahout.cf.taste.hadoop;
 
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.JobConf;
@@ -31,7 +33,6 @@
 import org.apache.mahout.cf.taste.recommender.Recommender;
 
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
@@ -77,13 +78,17 @@
   @Override
   public void configure(JobConf jobConf) {
     String dataModelFile = jobConf.get(DATA_MODEL_FILE);
+    String recommenderClassName = jobConf.get(RECOMMENDER_CLASS_NAME);
     FileDataModel fileDataModel;
     try {
-      fileDataModel = new FileDataModel(new File(dataModelFile));
-    } catch (FileNotFoundException fnfe) {
-      throw new RuntimeException(fnfe);
+      FileSystem fs = FileSystem.get(jobConf);
+      File tempDataFile = File.createTempFile("mahout-taste-hadoop", "txt");
+      tempDataFile.deleteOnExit();
+      fs.copyToLocalFile(new Path(dataModelFile), new 
Path(tempDataFile.getAbsolutePath()));
+      fileDataModel = new FileDataModel(tempDataFile);
+    } catch (IOException ioe) {
+      throw new RuntimeException(ioe);
     }
-    String recommenderClassName = jobConf.get(RECOMMENDER_CLASS_NAME);
     try {
       Class<? extends Recommender> recommenderClass =
           (Class<? extends Recommender>) Class.forName(recommenderClassName);