Author: tomwhite
Date: Mon Nov 15 17:16:25 2010
New Revision: 1035357
URL: http://svn.apache.org/viewvc?rev=1035357&view=rev
Log:
MAPREDUCE-1784. IFile should check for null compressor. Contributed by Eli
Collins.
Added:
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIFile.java
(with props)
Modified:
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IFile.java
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IFile.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IFile.java?rev=1035357&r1=1035356&r2=1035357&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IFile.java
(original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/IFile.java Mon Nov
15 17:16:25 2010
@@ -42,6 +42,9 @@ import org.apache.hadoop.io.compress.Dec
import org.apache.hadoop.io.serializer.SerializationFactory;
import org.apache.hadoop.io.serializer.Serializer;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
/**
* <code>IFile</code> is the simple <key-len, value-len, key, value> format
* for the intermediate map-outputs in Map-Reduce.
@@ -52,7 +55,7 @@ import org.apache.hadoop.io.serializer.S
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class IFile {
-
+ private static final Log LOG = LogFactory.getLog(IFile.class);
public static final int EOF_MARKER = -1; // End of File Marker
/**
@@ -107,13 +110,17 @@ public class IFile {
this.checksumOut = new IFileOutputStream(out);
this.rawOut = out;
this.start = this.rawOut.getPos();
-
if (codec != null) {
this.compressor = CodecPool.getCompressor(codec);
- this.compressor.reset();
- this.compressedOut = codec.createOutputStream(checksumOut, compressor);
- this.out = new FSDataOutputStream(this.compressedOut, null);
- this.compressOutput = true;
+ if (this.compressor != null) {
+ this.compressor.reset();
+ this.compressedOut = codec.createOutputStream(checksumOut,
compressor);
+ this.out = new FSDataOutputStream(this.compressedOut, null);
+ this.compressOutput = true;
+ } else {
+ LOG.warn("Could not obtain compressor from CodecPool");
+ this.out = new FSDataOutputStream(checksumOut,null);
+ }
} else {
this.out = new FSDataOutputStream(checksumOut,null);
}
@@ -335,7 +342,12 @@ public class IFile {
checksumIn = new IFileInputStream(in,length);
if (codec != null) {
decompressor = CodecPool.getDecompressor(codec);
- this.in = codec.createInputStream(checksumIn, decompressor);
+ if (decompressor != null) {
+ this.in = codec.createInputStream(checksumIn, decompressor);
+ } else {
+ LOG.warn("Could not obtain decompressor from CodecPool");
+ this.in = checksumIn;
+ }
} else {
this.in = checksumIn;
}
Added:
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIFile.java
URL:
http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIFile.java?rev=1035357&view=auto
==============================================================================
---
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIFile.java
(added)
+++
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIFile.java
Mon Nov 15 17:16:25 2010
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.DefaultCodec;
+import org.apache.hadoop.io.compress.GzipCodec;
+
+import org.junit.Test;
+
+public class TestIFile {
+
+ @Test
+ /**
+ * Create an IFile.Writer using GzipCodec since this codec does not
+ * have a compressor when run via the tests (ie no native libraries).
+ */
+ public void testIFileWriterWithCodec() throws Exception {
+ Configuration conf = new Configuration();
+ FileSystem localFs = FileSystem.getLocal(conf);
+ FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
+ Path path = new Path(new Path("build/test.ifile"), "data");
+ DefaultCodec codec = new GzipCodec();
+ codec.setConf(conf);
+ IFile.Writer<Text, Text> writer =
+ new IFile.Writer<Text, Text>(conf, rfs, path, Text.class, Text.class,
+ codec, null);
+ writer.close();
+ }
+
+ @Test
+ /** Same as above but create a reader. */
+ public void testIFileReaderWithCodec() throws Exception {
+ Configuration conf = new Configuration();
+ FileSystem localFs = FileSystem.getLocal(conf);
+ FileSystem rfs = ((LocalFileSystem)localFs).getRaw();
+ Path path = new Path(new Path("build/test.ifile"), "data");
+ DefaultCodec codec = new GzipCodec();
+ codec.setConf(conf);
+ IFile.Reader<Text, Text> reader =
+ new IFile.Reader<Text, Text>(conf, rfs, path, codec, null);
+ reader.close();
+ }
+}
Propchange:
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestIFile.java
------------------------------------------------------------------------------
svn:eol-style = native
