Author: hashutosh
Date: Sun Jun 1 17:56:44 2014
New Revision: 1599020
URL: http://svn.apache.org/r1599020
Log:
HIVE-7137 : Add progressable to writer interfaces so they could report progress
while different operations are in progress (Sumit Kumar via Ashutosh Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Sun
Jun 1 17:56:44 2014
@@ -177,6 +177,7 @@ import org.apache.hadoop.mapred.RecordRe
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Shell;
@@ -1354,9 +1355,9 @@ public final class Utilities {
* @return output stream over the created sequencefile
*/
public static SequenceFile.Writer createSequenceWriter(JobConf jc,
FileSystem fs, Path file,
- Class<?> keyClass, Class<?> valClass) throws IOException {
+ Class<?> keyClass, Class<?> valClass, Progressable progressable) throws
IOException {
boolean isCompressed = FileOutputFormat.getCompressOutput(jc);
- return createSequenceWriter(jc, fs, file, keyClass, valClass,
isCompressed);
+ return createSequenceWriter(jc, fs, file, keyClass, valClass,
isCompressed, progressable);
}
/**
@@ -1376,7 +1377,8 @@ public final class Utilities {
* @return output stream over the created sequencefile
*/
public static SequenceFile.Writer createSequenceWriter(JobConf jc,
FileSystem fs, Path file,
- Class<?> keyClass, Class<?> valClass, boolean isCompressed) throws
IOException {
+ Class<?> keyClass, Class<?> valClass, boolean isCompressed, Progressable
progressable)
+ throws IOException {
CompressionCodec codec = null;
CompressionType compressionType = CompressionType.NONE;
Class codecClass = null;
@@ -1385,7 +1387,8 @@ public final class Utilities {
codecClass = FileOutputFormat.getOutputCompressorClass(jc,
DefaultCodec.class);
codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
}
- return (SequenceFile.createWriter(fs, jc, file, keyClass, valClass,
compressionType, codec));
+ return (SequenceFile.createWriter(fs, jc, file, keyClass, valClass,
compressionType, codec,
+ progressable));
}
@@ -1402,14 +1405,14 @@ public final class Utilities {
* @return output stream over the created rcfile
*/
public static RCFile.Writer createRCFileWriter(JobConf jc, FileSystem fs,
Path file,
- boolean isCompressed) throws IOException {
+ boolean isCompressed, Progressable progressable) throws IOException {
CompressionCodec codec = null;
Class<?> codecClass = null;
if (isCompressed) {
codecClass = FileOutputFormat.getOutputCompressorClass(jc,
DefaultCodec.class);
codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
}
- return new RCFile.Writer(fs, jc, file, null, codec);
+ return new RCFile.Writer(fs, jc, file, progressable, codec);
}
/**
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/PTFRowContainer.java
Sun Jun 1 17:56:44 2014
@@ -269,8 +269,8 @@ public class PTFRowContainer<Row extends
Properties tableProperties, Progressable progress) throws IOException {
FileSystem fs = finalOutPath.getFileSystem(jc);
- final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
- fs, finalOutPath, BytesWritable.class, valueClass, isCompressed);
+ final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
fs, finalOutPath,
+ BytesWritable.class, valueClass, isCompressed, progress);
return new PTFRecordWriter(outStream);
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveBinaryOutputFormat.java
Sun Jun 1 17:56:44 2014
@@ -64,7 +64,7 @@ public class HiveBinaryOutputFormat<K ex
Properties tableProperties, Progressable progress) throws IOException {
FileSystem fs = outPath.getFileSystem(jc);
- final OutputStream outStream = fs.create(outPath);
+ final OutputStream outStream = fs.create(outPath, progress);
return new RecordWriter() {
@Override
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java
Sun Jun 1 17:56:44 2014
@@ -77,8 +77,8 @@ public class HiveIgnoreKeyTextOutputForm
final int finalRowSeparator = rowSeparator;
FileSystem fs = outPath.getFileSystem(jc);
- final OutputStream outStream = Utilities.createCompressedStream(jc, fs
- .create(outPath), isCompressed);
+ final OutputStream outStream = Utilities.createCompressedStream(jc,
+ fs.create(outPath, progress), isCompressed);
return new RecordWriter() {
@Override
public void write(Writable r) throws IOException {
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveNullValueSequenceFileOutputFormat.java
Sun Jun 1 17:56:44 2014
@@ -53,8 +53,8 @@ public class HiveNullValueSequenceFileOu
Properties tableProperties, Progressable progress) throws IOException {
FileSystem fs = finalOutPath.getFileSystem(jc);
- final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
- fs, finalOutPath, HiveKey.class, NullWritable.class, isCompressed);
+ final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
fs, finalOutPath,
+ HiveKey.class, NullWritable.class, isCompressed, progress);
keyWritable = new HiveKey();
keyIsText = valueClass.equals(Text.class);
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java
Sun Jun 1 17:56:44 2014
@@ -61,8 +61,8 @@ public class HiveSequenceFileOutputForma
Properties tableProperties, Progressable progress) throws IOException {
FileSystem fs = finalOutPath.getFileSystem(jc);
- final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
- fs, finalOutPath, BytesWritable.class, valueClass, isCompressed);
+ final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
fs, finalOutPath,
+ BytesWritable.class, valueClass, isCompressed, progress);
return new RecordWriter() {
@Override
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java?rev=1599020&r1=1599019&r2=1599020&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileOutputFormat.java
Sun Jun 1 17:56:44 2014
@@ -131,9 +131,8 @@ public class RCFileOutputFormat extends
}
RCFileOutputFormat.setColumnNumber(jc, cols.length);
- final RCFile.Writer outWriter = Utilities.createRCFileWriter
- (jc, finalOutPath.getFileSystem(jc),
- finalOutPath, isCompressed);
+ final RCFile.Writer outWriter = Utilities.createRCFileWriter(jc,
+ finalOutPath.getFileSystem(jc), finalOutPath, isCompressed, progress);
return new org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter() {
@Override
