Author: cutting
Date: Thu Jul 12 14:08:22 2007
New Revision: 555768
URL: http://svn.apache.org/viewvc?view=rev&rev=555768
Log:
HADOOP-1570. Permit jobs to enable and disable use of Hadoop's native library.
Contributed by Arun.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/conf/hadoop-default.xml
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/SequenceFile.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/DefaultCodec.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/GzipCodec.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/LzoCodec.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/util/NativeCodeLoader.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/io/TestSequenceFile.java
lucene/hadoop/trunk/src/test/org/apache/hadoop/io/compress/TestCodec.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Jul 12 14:08:22 2007
@@ -343,6 +343,9 @@
106. HADOOP-1547. Provide examples for aggregate library.
(Runping Qi via tomwhite)
+107. HADOOP-1570. Permit jobs to enable and disable the use of
+ hadoop's native library. (Arun C Murthy via cutting)
+
Release 0.13.0 - 2007-06-08
Modified: lucene/hadoop/trunk/conf/hadoop-default.xml
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/conf/hadoop-default.xml?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/conf/hadoop-default.xml (original)
+++ lucene/hadoop/trunk/conf/hadoop-default.xml Thu Jul 12 14:08:22 2007
@@ -15,6 +15,11 @@
<description>A base for other temporary directories.</description>
</property>
+<property>
+ <name>hadoop.native.lib</name>
+ <value>true</value>
+ <description>Should native hadoop libraries, if present, be
used.</description>
+</property>
<!--- logging properties -->
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/SequenceFile.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/SequenceFile.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/SequenceFile.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/SequenceFile.java Thu Jul
12 14:08:22 2007
@@ -193,7 +193,7 @@
throws IOException {
if ((codec instanceof GzipCodec) &&
!NativeCodeLoader.isNativeCodeLoaded() &&
- !ZlibFactory.isNativeZlibLoaded()) {
+ !ZlibFactory.isNativeZlibLoaded(conf)) {
throw new IllegalArgumentException("SequenceFile doesn't work with " +
"GzipCodec without native-hadoop
code!");
}
@@ -234,7 +234,7 @@
Progressable progress, Metadata metadata) throws IOException {
if ((codec instanceof GzipCodec) &&
!NativeCodeLoader.isNativeCodeLoaded() &&
- !ZlibFactory.isNativeZlibLoaded()) {
+ !ZlibFactory.isNativeZlibLoaded(conf)) {
throw new IllegalArgumentException("SequenceFile doesn't work with " +
"GzipCodec without native-hadoop
code!");
}
@@ -295,7 +295,7 @@
throws IOException {
if (codec != null && (codec instanceof GzipCodec) &&
!NativeCodeLoader.isNativeCodeLoaded() &&
- !ZlibFactory.isNativeZlibLoaded()) {
+ !ZlibFactory.isNativeZlibLoaded(conf)) {
throw new IllegalArgumentException("SequenceFile doesn't work with " +
"GzipCodec without native-hadoop
code!");
}
@@ -336,7 +336,7 @@
throws IOException {
if (codec != null && (codec instanceof GzipCodec) &&
!NativeCodeLoader.isNativeCodeLoaded() &&
- !ZlibFactory.isNativeZlibLoaded()) {
+ !ZlibFactory.isNativeZlibLoaded(conf)) {
throw new IllegalArgumentException("SequenceFile doesn't work with " +
"GzipCodec without native-hadoop
code!");
}
@@ -375,7 +375,7 @@
throws IOException {
if ((codec instanceof GzipCodec) &&
!NativeCodeLoader.isNativeCodeLoaded() &&
- !ZlibFactory.isNativeZlibLoaded()) {
+ !ZlibFactory.isNativeZlibLoaded(conf)) {
throw new IllegalArgumentException("SequenceFile doesn't work with " +
"GzipCodec without native-hadoop
code!");
}
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/DefaultCodec.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/DefaultCodec.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/DefaultCodec.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/DefaultCodec.java
Thu Jul 12 14:08:22 2007
@@ -52,11 +52,11 @@
}
public Class getCompressorType() {
- return ZlibFactory.getZlibCompressorType();
+ return ZlibFactory.getZlibCompressorType(conf);
}
public Compressor createCompressor() {
- return ZlibFactory.getZlibCompressor();
+ return ZlibFactory.getZlibCompressor(conf);
}
public CompressionInputStream createInputStream(InputStream in)
@@ -73,11 +73,11 @@
}
public Class getDecompressorType() {
- return ZlibFactory.getZlibDecompressorType();
+ return ZlibFactory.getZlibDecompressorType(conf);
}
public Decompressor createDecompressor() {
- return ZlibFactory.getZlibDecompressor();
+ return ZlibFactory.getZlibDecompressor(conf);
}
public String getDefaultExtension() {
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/GzipCodec.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/GzipCodec.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/GzipCodec.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/GzipCodec.java
Thu Jul 12 14:08:22 2007
@@ -135,7 +135,7 @@
public CompressionOutputStream createOutputStream(OutputStream out)
throws IOException {
- return (ZlibFactory.isNativeZlibLoaded()) ?
+ return (ZlibFactory.isNativeZlibLoaded(conf)) ?
new CompressorStream(out, createCompressor(),
conf.getInt("io.file.buffer.size",
4*1024)) :
new GzipOutputStream(out);
@@ -153,7 +153,7 @@
}
public Compressor createCompressor() {
- return (ZlibFactory.isNativeZlibLoaded()) ?
+ return (ZlibFactory.isNativeZlibLoaded(conf)) ?
new
ZlibCompressor(ZlibCompressor.CompressionLevel.DEFAULT_COMPRESSION,
ZlibCompressor.CompressionStrategy.DEFAULT_STRATEGY,
ZlibCompressor.CompressionHeader.GZIP_FORMAT,
@@ -162,12 +162,12 @@
}
public Class getCompressorType() {
- return ZlibFactory.getZlibCompressorType();
+ return ZlibFactory.getZlibCompressorType(conf);
}
public CompressionInputStream createInputStream(InputStream in)
throws IOException {
- return (ZlibFactory.isNativeZlibLoaded()) ?
+ return (ZlibFactory.isNativeZlibLoaded(conf)) ?
new DecompressorStream(in, createDecompressor(),
conf.getInt("io.file.buffer.size",
4*1024)) :
@@ -185,14 +185,14 @@
}
public Decompressor createDecompressor() {
- return (ZlibFactory.isNativeZlibLoaded()) ?
+ return (ZlibFactory.isNativeZlibLoaded(conf)) ?
new
ZlibDecompressor(ZlibDecompressor.CompressionHeader.AUTODETECT_GZIP_ZLIB,
64*1024) :
null;
}
public Class getDecompressorType() {
- return ZlibFactory.getZlibDecompressorType();
+ return ZlibFactory.getZlibDecompressorType(conf);
}
public String getDefaultExtension() {
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/LzoCodec.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/LzoCodec.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/LzoCodec.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/LzoCodec.java
Thu Jul 12 14:08:22 2007
@@ -69,17 +69,18 @@
/**
* Check if native-lzo library is loaded & initialized.
*
+ * @param conf configuration
* @return <code>true</code> if native-lzo library is loaded & initialized;
* else <code>false</code>
*/
- public static boolean isNativeLzoLoaded() {
- return nativeLzoLoaded;
+ public static boolean isNativeLzoLoaded(Configuration conf) {
+ return nativeLzoLoaded && conf.getBoolean("hadoop.native.lib", true);
}
public CompressionOutputStream createOutputStream(OutputStream out)
throws IOException {
// Ensure native-lzo library is loaded & initialized
- if (!isNativeLzoLoaded()) {
+ if (!isNativeLzoLoaded(conf)) {
throw new RuntimeException("native-lzo library not available");
}
@@ -128,7 +129,7 @@
Compressor compressor)
throws IOException {
// Ensure native-lzo library is loaded & initialized
- if (!isNativeLzoLoaded()) {
+ if (!isNativeLzoLoaded(conf)) {
throw new RuntimeException("native-lzo library not available");
}
@@ -153,7 +154,7 @@
public Class getCompressorType() {
// Ensure native-lzo library is loaded & initialized
- if (!isNativeLzoLoaded()) {
+ if (!isNativeLzoLoaded(conf)) {
throw new RuntimeException("native-lzo library not available");
}
@@ -162,7 +163,7 @@
public Compressor createCompressor() {
// Ensure native-lzo library is loaded & initialized
- if (!isNativeLzoLoaded()) {
+ if (!isNativeLzoLoaded(conf)) {
throw new RuntimeException("native-lzo library not available");
}
@@ -181,7 +182,7 @@
public CompressionInputStream createInputStream(InputStream in)
throws IOException {
// Ensure native-lzo library is loaded & initialized
- if (!isNativeLzoLoaded()) {
+ if (!isNativeLzoLoaded(conf)) {
throw new IOException("native-lzo library not available");
}
@@ -204,7 +205,7 @@
Decompressor decompressor)
throws IOException {
// Ensure native-lzo library is loaded & initialized
- if (!isNativeLzoLoaded()) {
+ if (!isNativeLzoLoaded(conf)) {
throw new RuntimeException("native-lzo library not available");
}
@@ -215,7 +216,7 @@
public Class getDecompressorType() {
// Ensure native-lzo library is loaded & initialized
- if (!isNativeLzoLoaded()) {
+ if (!isNativeLzoLoaded(conf)) {
throw new RuntimeException("native-lzo library not available");
}
@@ -224,7 +225,7 @@
public Decompressor createDecompressor() {
// Ensure native-lzo library is loaded & initialized
- if (!isNativeLzoLoaded()) {
+ if (!isNativeLzoLoaded(conf)) {
throw new RuntimeException("native-lzo library not available");
}
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
---
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
(original)
+++
lucene/hadoop/trunk/src/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
Thu Jul 12 14:08:22 2007
@@ -20,10 +20,10 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.compress.Compressor;
import org.apache.hadoop.io.compress.Decompressor;
import org.apache.hadoop.util.NativeCodeLoader;
-import org.apache.hadoop.util.StringUtils;
/**
* A collection of factories to create the right
@@ -50,54 +50,58 @@
}
/**
- * Check if native-zlib code is loaded and initialized correctly.
+ * Check if native-zlib code is loaded & initialized correctly and
+ * can be loaded for this job.
*
- * @return <code>true</code> if native-zlib is loaded and initialized,
- * else <code>false</code>
+ * @param conf configuration
+ * @return <code>true</code> if native-zlib is loaded & initialized
+ * and can be loaded for this job, else <code>false</code>
*/
- public static boolean isNativeZlibLoaded() {
- return nativeZlibLoaded;
+ public static boolean isNativeZlibLoaded(Configuration conf) {
+ return nativeZlibLoaded && conf.getBoolean("hadoop.native.lib", true);
}
/**
* Return the appropriate type of the zlib compressor.
*
+ * @param conf configuration
* @return the appropriate type of the zlib compressor.
*/
- public static Class getZlibCompressorType() {
- return (nativeZlibLoaded) ?
+ public static Class getZlibCompressorType(Configuration conf) {
+ return (isNativeZlibLoaded(conf)) ?
ZlibCompressor.class : BuiltInZlibDeflater.class;
}
/**
* Return the appropriate implementation of the zlib compressor.
*
+ * @param conf configuration
* @return the appropriate implementation of the zlib compressor.
*/
- public static Compressor getZlibCompressor() {
- LOG.info("Creating a new ZlibCompressor");
- return (nativeZlibLoaded) ?
+ public static Compressor getZlibCompressor(Configuration conf) {
+ return (isNativeZlibLoaded(conf)) ?
new ZlibCompressor() : new BuiltInZlibDeflater();
}
/**
* Return the appropriate type of the zlib decompressor.
*
+ * @param conf configuration
* @return the appropriate type of the zlib decompressor.
*/
- public static Class getZlibDecompressorType() {
- return (nativeZlibLoaded) ?
+ public static Class getZlibDecompressorType(Configuration conf) {
+ return (isNativeZlibLoaded(conf)) ?
ZlibDecompressor.class : BuiltInZlibInflater.class;
}
/**
* Return the appropriate implementation of the zlib decompressor.
*
+ * @param conf configuration
* @return the appropriate implementation of the zlib decompressor.
*/
- public static Decompressor getZlibDecompressor() {
- LOG.info("Creating a new ZlibDecompressor");
- return (nativeZlibLoaded) ?
+ public static Decompressor getZlibDecompressor(Configuration conf) {
+ return (isNativeZlibLoaded(conf)) ?
new ZlibDecompressor() : new BuiltInZlibInflater();
}
Modified:
lucene/hadoop/trunk/src/java/org/apache/hadoop/util/NativeCodeLoader.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/NativeCodeLoader.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/NativeCodeLoader.java
(original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/NativeCodeLoader.java
Thu Jul 12 14:08:22 2007
@@ -19,6 +19,7 @@
package org.apache.hadoop.util;
import org.apache.commons.logging.*;
+import org.apache.hadoop.mapred.JobConf;
/**
* A helper to load the native hadoop code i.e. libhadoop.so.
@@ -60,6 +61,28 @@
*/
public static boolean isNativeCodeLoaded() {
return nativeCodeLoaded;
+ }
+
+ /**
+ * Return if native hadoop libraries, if present, can be used for this job.
+ * @param jobConf job configuration
+ *
+ * @return <code>true</code> if native hadoop libraries, if present, can be
+ * used for this job; <code>false</code> otherwise.
+ */
+ public boolean getLoadNativeLibraries(JobConf jobConf) {
+ return jobConf.getBoolean("hadoop.native.lib", true);
+ }
+
+ /**
+ * Set if native hadoop libraries, if present, can be used for this job.
+ *
+ * @param jobConf job configuration
+ * @param loadNativeLibraries can native hadoop libraries be loaded
+ */
+ public void setLoadNativeLibraries(JobConf jobConf,
+ boolean loadNativeLibraries) {
+ jobConf.setBoolean("hadoop.native.lib", loadNativeLibraries);
}
}
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/io/TestSequenceFile.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/io/TestSequenceFile.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/io/TestSequenceFile.java
(original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/io/TestSequenceFile.java Thu
Jul 12 14:08:22 2007
@@ -49,7 +49,7 @@
}
public void testLzoSequenceFile() throws Exception {
- if (LzoCodec.isNativeLzoLoaded()) {
+ if (LzoCodec.isNativeLzoLoaded(conf)) {
LOG.info("Testing SequenceFile with LzoCodec");
CompressionCodec lzoCodec = null;
try {
Modified:
lucene/hadoop/trunk/src/test/org/apache/hadoop/io/compress/TestCodec.java
URL:
http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/io/compress/TestCodec.java?view=diff&rev=555768&r1=555767&r2=555768
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/io/compress/TestCodec.java
(original)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/io/compress/TestCodec.java
Thu Jul 12 14:08:22 2007
@@ -40,28 +40,29 @@
private static final Log LOG=
LogFactory.getLog("org.apache.hadoop.io.compress.TestCodec");
+ private Configuration conf = new Configuration();
private int count = 10000;
private int seed = new Random().nextInt();
public void testDefaultCodec() throws IOException {
- codecTest(seed, count, "org.apache.hadoop.io.compress.DefaultCodec");
+ codecTest(conf, seed, count, "org.apache.hadoop.io.compress.DefaultCodec");
}
public void testGzipCodec() throws IOException {
- codecTest(seed, count, "org.apache.hadoop.io.compress.GzipCodec");
+ codecTest(conf, seed, count, "org.apache.hadoop.io.compress.GzipCodec");
}
public void testLzoCodec() throws IOException {
- if (LzoCodec.isNativeLzoLoaded()) {
- codecTest(seed, count, "org.apache.hadoop.io.compress.LzoCodec");
+ if (LzoCodec.isNativeLzoLoaded(conf)) {
+ codecTest(conf, seed, count, "org.apache.hadoop.io.compress.LzoCodec");
}
}
- private static void codecTest(int seed, int count, String codecClass)
+ private static void codecTest(Configuration conf, int seed, int count,
+ String codecClass)
throws IOException {
// Create the codec
- Configuration conf = new Configuration();
CompressionCodec codec = null;
try {
codec = (CompressionCodec)
@@ -144,8 +145,9 @@
}
}
+ Configuration conf = new Configuration();
int seed = 0;
- codecTest(seed, count, codecClass);
+ codecTest(conf, seed, count, codecClass);
} catch (Exception e) {
System.err.println("Caught: " + e);
e.printStackTrace();
