Author: tomwhite
Date: Thu Apr 19 19:20:31 2012
New Revision: 1328083
URL: http://svn.apache.org/viewvc?rev=1328083&view=rev
Log:
HADOOP-7350. Use ServiceLoader to discover compression codec classes.
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1328083&r1=1328082&r2=1328083&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Apr
19 19:20:31 2012
@@ -360,6 +360,9 @@ Release 2.0.0 - UNRELEASED
HADOOP-8282. start-all.sh refers incorrectly start-dfs.sh
existence for starting start-yarn.sh. (Devaraj K via eli)
+ HADOOP-7350. Use ServiceLoader to discover compression codec classes.
+ (tomwhite)
+
BREAKDOWN OF HADOOP-7454 SUBTASKS
HADOOP-7455. HA: Introduce HA Service Protocol Interface. (suresh)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java?rev=1328083&r1=1328082&r2=1328083&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
Thu Apr 19 19:20:31 2012
@@ -36,6 +36,9 @@ public class CompressionCodecFactory {
public static final Log LOG =
LogFactory.getLog(CompressionCodecFactory.class.getName());
+
+ private static final ServiceLoader<CompressionCodec> CODEC_PROVIDERS =
+ ServiceLoader.load(CompressionCodec.class);
/**
* A map from the reversed filename suffixes to the codecs.
@@ -95,16 +98,23 @@ public class CompressionCodecFactory {
}
/**
- * Get the list of codecs listed in the configuration
+ * Get the list of codecs discovered via a Java ServiceLoader, or
+ * listed in the configuration. Codecs specified in configuration come
+ * later in the returned list, and are considered to override those
+ * from the ServiceLoader.
* @param conf the configuration to look in
- * @return a list of the Configuration classes or null if the attribute
- * was not set
+ * @return a list of the {@link CompressionCodec} classes
*/
public static List<Class<? extends CompressionCodec>>
getCodecClasses(Configuration conf) {
+ List<Class<? extends CompressionCodec>> result
+ = new ArrayList<Class<? extends CompressionCodec>>();
+ // Add codec classes discovered via service loading
+ for (CompressionCodec codec : CODEC_PROVIDERS) {
+ result.add(codec.getClass());
+ }
+ // Add codec classes from configuration
String codecsString = conf.get("io.compression.codecs");
if (codecsString != null) {
- List<Class<? extends CompressionCodec>> result
- = new ArrayList<Class<? extends CompressionCodec>>();
StringTokenizer codecSplit = new StringTokenizer(codecsString, ",");
while (codecSplit.hasMoreElements()) {
String codecSubstring = codecSplit.nextToken();
@@ -123,14 +133,14 @@ public class CompressionCodecFactory {
}
}
}
- return result;
- } else {
- return null;
}
+ return result;
}
/**
- * Sets a list of codec classes in the configuration.
+ * Sets a list of codec classes in the configuration. In addition to any
+ * classes specified using this method, {@link CompressionCodec} classes on
+ * the classpath are discovered using a Java ServiceLoader.
* @param conf the configuration to modify
* @param classes the list of classes to set
*/
@@ -151,21 +161,19 @@ public class CompressionCodecFactory {
/**
* Find the codecs specified in the config value io.compression.codecs
- * and register them. Defaults to gzip and zip.
+ * and register them. Defaults to gzip and deflate.
*/
public CompressionCodecFactory(Configuration conf) {
codecs = new TreeMap<String, CompressionCodec>();
codecsByClassName = new HashMap<String, CompressionCodec>();
codecsByName = new HashMap<String, CompressionCodec>();
List<Class<? extends CompressionCodec>> codecClasses =
getCodecClasses(conf);
- if (codecClasses == null) {
+ if (codecClasses == null || codecClasses.isEmpty()) {
addCodec(new GzipCodec());
addCodec(new DefaultCodec());
} else {
- Iterator<Class<? extends CompressionCodec>> itr =
codecClasses.iterator();
- while (itr.hasNext()) {
- CompressionCodec codec = ReflectionUtils.newInstance(itr.next(), conf);
- addCodec(codec);
+ for (Class<? extends CompressionCodec> codecClass : codecClasses) {
+ addCodec(ReflectionUtils.newInstance(codecClass, conf));
}
}
}
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec?rev=1328083&view=auto
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec
(added)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec
Thu Apr 19 19:20:31 2012
@@ -0,0 +1,20 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.hadoop.io.compress.BZip2Codec
+org.apache.hadoop.io.compress.DefaultCodec
+org.apache.hadoop.io.compress.DeflateCodec
+org.apache.hadoop.io.compress.GzipCodec
+org.apache.hadoop.io.compress.Lz4Codec
+org.apache.hadoop.io.compress.SnappyCodec
+
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1328083&r1=1328082&r2=1328083&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
Thu Apr 19 19:20:31 2012
@@ -268,9 +268,11 @@
<property>
<name>io.compression.codecs</name>
-
<value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
- <description>A list of the compression codec classes that can be used
- for compression/decompression.</description>
+ <value></value>
+ <description>A comma-separated list of the compression codec classes that can
+ be used for compression/decompression. In addition to any classes specified
+ with this property (which take precedence), codec classes on the classpath
+ are discovered using a Java ServiceLoader.</description>
</property>
<property>
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java?rev=1328083&r1=1328082&r2=1328083&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
Thu Apr 19 19:20:31 2012
@@ -101,6 +101,12 @@ public class TestCodecFactory extends Te
}
}
+ private static class NewGzipCodec extends BaseCodec {
+ public String getDefaultExtension() {
+ return ".gz";
+ }
+ }
+
/**
* Returns a factory for a given set of codecs
* @param classes the codec classes to include
@@ -167,32 +173,43 @@ public class TestCodecFactory extends Te
checkCodec("default factory for deflate codec", DeflateCodec.class, codec);
factory = setClasses(new Class[0]);
+ // gz, bz2, snappy, lz4 are picked up by service loader, but bar isn't
codec = factory.getCodec(new Path("/tmp/foo.bar"));
- assertEquals("empty codec bar codec", null, codec);
+ assertEquals("empty factory bar codec", null, codec);
codec = factory.getCodecByClassName(BarCodec.class.getCanonicalName());
- assertEquals("empty codec bar codec", null, codec);
+ assertEquals("empty factory bar codec", null, codec);
codec = factory.getCodec(new Path("/tmp/foo.gz"));
- assertEquals("empty codec gz codec", null, codec);
+ checkCodec("empty factory gz codec", GzipCodec.class, codec);
codec = factory.getCodecByClassName(GzipCodec.class.getCanonicalName());
- assertEquals("empty codec gz codec", null, codec);
+ checkCodec("empty factory gz codec", GzipCodec.class, codec);
codec = factory.getCodec(new Path("/tmp/foo.bz2"));
- assertEquals("empty factory for .bz2", null, codec);
+ checkCodec("empty factory for .bz2", BZip2Codec.class, codec);
codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName());
- assertEquals("empty factory for bzip2 codec", null, codec);
+ checkCodec("empty factory for bzip2 codec", BZip2Codec.class, codec);
+
+ codec = factory.getCodec(new Path("/tmp/foo.snappy"));
+ checkCodec("empty factory snappy codec", SnappyCodec.class, codec);
+ codec = factory.getCodecByClassName(SnappyCodec.class.getCanonicalName());
+ checkCodec("empty factory snappy codec", SnappyCodec.class, codec);
+
+ codec = factory.getCodec(new Path("/tmp/foo.lz4"));
+ checkCodec("empty factory lz4 codec", Lz4Codec.class, codec);
+ codec = factory.getCodecByClassName(Lz4Codec.class.getCanonicalName());
+ checkCodec("empty factory lz4 codec", Lz4Codec.class, codec);
factory = setClasses(new Class[]{BarCodec.class, FooCodec.class,
FooBarCodec.class});
codec = factory.getCodec(new Path("/tmp/.foo.bar.gz"));
- assertEquals("full factory gz codec", null, codec);
+ checkCodec("full factory gz codec", GzipCodec.class, codec);
codec = factory.getCodecByClassName(GzipCodec.class.getCanonicalName());
- assertEquals("full codec gz codec", null, codec);
+ checkCodec("full codec gz codec", GzipCodec.class, codec);
codec = factory.getCodec(new Path("/tmp/foo.bz2"));
- assertEquals("full factory for .bz2", null, codec);
+ checkCodec("full factory for .bz2", BZip2Codec.class, codec);
codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName());
- assertEquals("full codec bzip2 codec", null, codec);
+ checkCodec("full codec bzip2 codec", BZip2Codec.class, codec);
codec = factory.getCodec(new Path("/tmp/foo.bar"));
checkCodec("full factory bar codec", BarCodec.class, codec);
@@ -220,5 +237,11 @@ public class TestCodecFactory extends Te
checkCodec("full factory foo codec", FooCodec.class, codec);
codec = factory.getCodecByName("FOO");
checkCodec("full factory foo codec", FooCodec.class, codec);
+
+ factory = setClasses(new Class[]{NewGzipCodec.class});
+ codec = factory.getCodec(new Path("/tmp/foo.gz"));
+ checkCodec("overridden factory for .gz", NewGzipCodec.class, codec);
+ codec = factory.getCodecByClassName(NewGzipCodec.class.getCanonicalName());
+ checkCodec("overridden factory for gzip codec", NewGzipCodec.class, codec);
}
}
