Author: cutting
Date: Thu Mar 23 16:28:01 2006
New Revision: 388306
URL: http://svn.apache.org/viewcvs?rev=388306&view=rev
Log:
Move checking of output directory existence from JobClient to OutputFormat, so
that it can be overridden. Add a base class for OutputFormat that implements
this new method.
Added:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java?rev=388306&r1=388305&r2=388306&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobClient.java Thu
Mar 23 16:28:01 2006
@@ -256,15 +256,8 @@
job.setWorkingDirectory(fileSys.getWorkingDirectory().toString());
}
- // Ensure that the output directory is set and not already there
- File outDir = job.getOutputDir();
- if (outDir == null && job.getNumReduceTasks() != 0) {
- throw new IOException("Output directory not set in JobConf.");
- }
- if (outDir != null && fs.exists(outDir)) {
- throw new IOException("Output directory " + outDir +
- " already exists.");
- }
+ // Check the output specification
+ job.getOutputFormat().checkOutputSpecs(fs, job);
// Write job file to JobTracker's fs
FSDataOutputStream out = fileSys.create(submitJobFile);
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java?rev=388306&r1=388305&r2=388306&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapFileOutputFormat.java
Thu Mar 23 16:28:01 2006
@@ -28,7 +28,7 @@
import org.apache.hadoop.conf.Configuration;
/** An [EMAIL PROTECTED] OutputFormat} that writes [EMAIL PROTECTED]
MapFile}s. */
-public class MapFileOutputFormat implements OutputFormat {
+public class MapFileOutputFormat extends OutputFormatBase {
public RecordWriter getRecordWriter(FileSystem fs, JobConf job,
String name) throws IOException {
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java?rev=388306&r1=388305&r2=388306&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormat.java Thu
Mar 23 16:28:01 2006
@@ -33,5 +33,15 @@
*/
RecordWriter getRecordWriter(FileSystem fs, JobConf job, String name)
throws IOException;
+
+ /** Check whether the output specification for a job is appropriate. Called
+ * when a job is submitted. Typically checks that it does not already exist,
+ * throwing an exception when it already exists, so that output is not
+ * overwritten.
+ *
+ * @param job the job whose output will be written
+ * @throws IOException when output should not be attempted
+ */
+ void checkOutputSpecs(FileSystem fs, JobConf job) throws IOException;
}
Added:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java?rev=388306&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java
(added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/OutputFormatBase.java
Thu Mar 23 16:28:01 2006
@@ -0,0 +1,43 @@
+/**
+ * Copyright 2006 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.io.File;
+
+import org.apache.hadoop.fs.FileSystem;
+
+/** A base class for [EMAIL PROTECTED] OutputFormat}. */
+public abstract class OutputFormatBase implements OutputFormat {
+ public abstract RecordWriter getRecordWriter(FileSystem fs,
+ JobConf job, String name)
+ throws IOException;
+
+ public void checkOutputSpecs(FileSystem fs, JobConf job) throws IOException {
+ // Ensure that the output directory is set and not already there
+ File outDir = job.getOutputDir();
+ if (outDir == null && job.getNumReduceTasks() != 0) {
+ throw new IOException("Output directory not set in JobConf.");
+ }
+ if (outDir != null && fs.exists(outDir)) {
+ throw new IOException("Output directory " + outDir +
+ " already exists.");
+ }
+ }
+
+}
+
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java?rev=388306&r1=388305&r2=388306&view=diff
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
Thu Mar 23 16:28:01 2006
@@ -28,7 +28,7 @@
import org.apache.hadoop.conf.Configuration;
/** An [EMAIL PROTECTED] OutputFormat} that writes [EMAIL PROTECTED]
SequenceFile}s. */
-public class SequenceFileOutputFormat implements OutputFormat {
+public class SequenceFileOutputFormat extends OutputFormatBase {
public RecordWriter getRecordWriter(FileSystem fs, JobConf job,
String name) throws IOException {
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java
URL:
http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java?rev=388306&r1=388305&r2=388306&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TextOutputFormat.java
Thu Mar 23 16:28:01 2006
@@ -26,7 +26,7 @@
import org.apache.hadoop.io.Writable;
/** An [EMAIL PROTECTED] OutputFormat} that writes plain text files. */
-public class TextOutputFormat implements OutputFormat {
+public class TextOutputFormat extends OutputFormatBase {
public RecordWriter getRecordWriter(FileSystem fs, JobConf job,
String name) throws IOException {
