Author: toffer
Date: Fri Sep 21 18:47:01 2012
New Revision: 1388613
URL: http://svn.apache.org/viewvc?rev=1388613&view=rev
Log:
addenum to HCAT-490, added extra comment
Modified:
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
Modified:
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java?rev=1388613&r1=1388612&r2=1388613&view=diff
==============================================================================
---
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
(original)
+++
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
Fri Sep 21 18:47:01 2012
@@ -195,6 +195,13 @@ class FileRecordWriterContainer extends
//create base OutputFormat
org.apache.hadoop.mapred.OutputFormat baseOF =
ReflectionUtils.newInstance(storageHandler.getOutputFormatClass(),
currTaskContext.getJobConf());
+
+ //We are skipping calling checkOutputSpecs() for each partition
+ //As it can throw a FileAlreadyExistsException when more than
one mapper is writing to a partition
+ //See HCATALOG-490, also to avoid contacting the namenode for
each new FileOutputFormat instance
+ //In general this should be ok for most FileOutputFormat
implementations
+ //but may become an issue for cases when the method is used to
perform other setup tasks
+
//get Output Committer
org.apache.hadoop.mapred.OutputCommitter baseOutputCommitter =
currTaskContext.getJobConf().getOutputCommitter();
//create currJobContext the latest so it gets all the config
changes
