Author: cutting
Date: Tue Aug 14 14:47:16 2007
New Revision: 565932
URL: http://svn.apache.org/viewvc?view=rev&rev=565932
Log:
HADOOP-1716. Fix Pipes wordcount example to remove the 'file:' schema from its
output path. Contributed by Owen.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/examples/pipes/impl/wordcount-nopipe.cc
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/pipes/WordCountInputFormat.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=565932&r1=565931&r2=565932
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Tue Aug 14 14:47:16 2007
@@ -523,6 +523,10 @@
153. HADOOP-1698. Fix performance problems on map output sorting for jobs
with large numbers of reduces. (Devaraj Das via omalley)
+154. HADOOP-1716. Fix a Pipes wordcount example to remove the 'file:'
+ schema from its output path. (omalley via cutting)
+
+
Release 0.13.0 - 2007-06-08
1. HADOOP-1047. Fix TestReplication to succeed more reliably.
Modified: lucene/hadoop/trunk/src/examples/pipes/impl/wordcount-nopipe.cc
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/examples/pipes/impl/wordcount-nopipe.cc?view=diff&rev=565932&r1=565931&r2=565932
==============================================================================
--- lucene/hadoop/trunk/src/examples/pipes/impl/wordcount-nopipe.cc (original)
+++ lucene/hadoop/trunk/src/examples/pipes/impl/wordcount-nopipe.cc Tue Aug 14
14:47:16 2007
@@ -87,9 +87,15 @@
const HadoopPipes::JobConf* job = context.getJobConf();
int part = job->getInt("mapred.task.partition");
std::string outDir = job->get("mapred.output.dir");
+ // remove the file: schema substring
+ std::string::size_type posn = outDir.find(":");
+ HADOOP_ASSERT(posn != std::string::npos,
+ "no schema found in output dir: " + outDir);
+ outDir.erase(0, posn+1);
mkdir(outDir.c_str(), 0777);
std::string outFile = outDir + "/part-" + HadoopUtils::toString(part);
file = fopen(outFile.c_str(), "wt");
+ HADOOP_ASSERT(file != NULL, "can't open file for writing: " + outFile);
}
~WordCountWriter() {
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java?view=diff&rev=565932&r1=565931&r2=565932
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
(original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
Tue Aug 14 14:47:16 2007
@@ -150,7 +150,8 @@
JobConf job = mr.createJobConf();
job.setInputFormat(WordCountInputFormat.class);
FileSystem local = FileSystem.getLocal(job);
- Path testDir = new Path(System.getProperty("test.build.data"), "pipes");
+ Path testDir = new Path("file:" + System.getProperty("test.build.data"),
+ "pipes");
Path inDir = new Path(testDir, "input");
Path outDir = new Path(testDir, "output");
Path wordExec = new Path("/testing/bin/application");
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/pipes/WordCountInputFormat.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/pipes/WordCountInputFormat.java?view=diff&rev=565932&r1=565931&r2=565932
==============================================================================
---
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/pipes/WordCountInputFormat.java
(original)
+++
lucene/hadoop/trunk/src/test/org/apache/hadoop/mapred/pipes/WordCountInputFormat.java
Tue Aug 14 14:47:16 2007
@@ -35,7 +35,7 @@
private String filename;
WordCountInputSplit() { }
WordCountInputSplit(Path filename) {
- this.filename = filename.toString();
+ this.filename = filename.toUri().getPath();
}
public void write(DataOutput out) throws IOException {
Text.writeString(out, filename);
