Author: seanzhong
Date: Wed Aug 6 07:32:49 2014
New Revision: 1616115
URL: http://svn.apache.org/r1616115
Log:
MAPREDUCE-5976. native-task: should not fail to build if snappy is missing
(Manu Zhang)
Removed:
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/SnappyUtil.java
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/snappy-c.h
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/snappy.h
Modified:
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
Modified:
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt?rev=1616115&r1=1616114&r2=1616115&view=diff
==============================================================================
---
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
(original)
+++
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
Wed Aug 6 07:32:49 2014
@@ -10,3 +10,4 @@ MAPREDUCE-5991. native-task should not r
MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin Chang)
+MAPREDUCE-5976. native-task: should not fail to build if snappy is missing
(Manu Zhang)
\ No newline at end of file
Modified:
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt?rev=1616115&r1=1616114&r2=1616115&view=diff
==============================================================================
---
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
(original)
+++
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
Wed Aug 6 07:32:49 2014
@@ -122,7 +122,6 @@ CHECK_FUNCTION_EXISTS(memset HAVE_MEMSET
CHECK_FUNCTION_EXISTS(strchr HAVE_STRCHR)
CHECK_FUNCTION_EXISTS(strtoul HAVE_STRTOUL)
-
SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
set_find_shared_library_version("1")
find_library(SNAPPY_LIBRARY
@@ -139,6 +138,7 @@ if (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DI
set(SNAPPY_SOURCE_FILES
"${D}/src/codec/SnappyCodec.cc")
else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+ set(SNAPPY_LIBRARY "")
set(SNAPPY_INCLUDE_DIR "")
set(SNAPPY_SOURCE_FILES "")
IF(REQUIRE_SNAPPY)
@@ -146,6 +146,8 @@ else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_
ENDIF(REQUIRE_SNAPPY)
endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+
include_directories(
${GENERATED_JAVAH}
${D}
@@ -154,15 +156,11 @@ include_directories(
${D}/src/lib
${D}/test
${CMAKE_CURRENT_SOURCE_DIR}
- #${CMAKE_CURRENT_SOURCE_DIR}/src
${CMAKE_BINARY_DIR}
${JNI_INCLUDE_DIRS}
${SNAPPY_INCLUDE_DIR}
)
-#SET(CMAKE_SOURCE_DIR
"/cygdrive/c/Users/tianlunz/repo/hadoop-2.2.0-src/hadoop-common-project/hadoop-common/src")
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
-
SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
Modified:
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java?rev=1616115&r1=1616114&r2=1616115&view=diff
==============================================================================
---
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
(original)
+++
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
Wed Aug 6 07:32:49 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapred.nativet
import java.io.File;
import java.io.IOException;
+import com.google.common.base.Charsets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
@@ -80,29 +81,22 @@ public class NativeMapOutputCollectorDel
Class comparatorClass = job.getClass(MRJobConfig.KEY_COMPARATOR, null,
RawComparator.class);
if (comparatorClass != null && !Platforms.define(comparatorClass)) {
- String message = "Native output collector don't support customized java
comparator "
+ String message = "Native output collector doesn't support customized
java comparator "
+ job.get(MRJobConfig.KEY_COMPARATOR);
LOG.error(message);
throw new InvalidJobConfException(message);
}
- if (job.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false) == true) {
- if (!isCodecSupported(job.get(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC))) {
- String message = "Native output collector don't support compression
codec "
- + job.get(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC) + ", We support
Gzip, Lz4, snappy";
- LOG.error(message);
- throw new InvalidJobConfException(message);
- }
- }
+
if (!QuickSort.class.getName().equals(job.get(Constants.MAP_SORT_CLASS))) {
- String message = "Native-Task don't support sort class " +
job.get(Constants.MAP_SORT_CLASS);
+ String message = "Native-Task doesn't support sort class " +
job.get(Constants.MAP_SORT_CLASS);
LOG.error(message);
throw new InvalidJobConfException(message);
}
if (job.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY, false) == true) {
- String message = "Native-Task don't support secure shuffle";
+ String message = "Native-Task doesn't support secure shuffle";
LOG.error(message);
throw new InvalidJobConfException(message);
}
@@ -116,7 +110,7 @@ public class NativeMapOutputCollectorDel
LOG.error(message);
throw new InvalidJobConfException(message);
} else if (!Platforms.support(keyCls.getName(), serializer, job)) {
- String message = "Native output collector don't support this key, this
key is not comparable in native "
+ String message = "Native output collector doesn't support this key,
this key is not comparable in native "
+ keyCls.getName();
LOG.error(message);
throw new InvalidJobConfException(message);
@@ -129,6 +123,14 @@ public class NativeMapOutputCollectorDel
final boolean ret = NativeRuntime.isNativeLibraryLoaded();
if (ret) {
+ if (job.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false)) {
+ String codec = job.get(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC);
+ if
(!NativeRuntime.supportsCompressionCodec(codec.getBytes(Charsets.UTF_8))) {
+ String message = "Native output collector doesn't support
compression codec " + codec;
+ LOG.error(message);
+ throw new InvalidJobConfException(message);
+ }
+ }
NativeRuntime.configure(job);
final long updateInterval =
job.getLong(Constants.NATIVE_STATUS_UPDATE_INTERVAL,
@@ -159,12 +161,4 @@ public class NativeMapOutputCollectorDel
LOG.info("Native output collector can be successfully enabled!");
}
- private boolean isCodecSupported(String string) {
- if ("org.apache.hadoop.io.compress.SnappyCodec".equals(string)
- || "org.apache.hadoop.io.compress.GzipCodec".equals(string)
- || "org.apache.hadoop.io.compress.Lz4Codec".equals(string)) {
- return true;
- }
- return false;
- }
}
Modified:
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java?rev=1616115&r1=1616114&r2=1616115&view=diff
==============================================================================
---
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
(original)
+++
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
Wed Aug 6 07:32:49 2014
@@ -31,7 +31,6 @@ import org.apache.hadoop.io.LongWritable
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.Task.TaskReporter;
import org.apache.hadoop.mapred.nativetask.util.ConfigUtil;
-import org.apache.hadoop.mapred.nativetask.util.SnappyUtil;
import org.apache.hadoop.util.VersionInfo;
/**
@@ -48,11 +47,6 @@ public class NativeRuntime {
static {
try {
- if (false == SnappyUtil.isNativeSnappyLoaded(conf)) {
- throw new IOException("Snappy library cannot be loaded");
- } else {
- LOG.info("Snappy native library is available");
- }
System.loadLibrary("nativetask");
LOG.info("Nativetask JNI library loaded.");
nativeLibraryLoaded = true;
@@ -162,6 +156,11 @@ public class NativeRuntime {
********************************************************/
/**
+ * Check whether the native side has compression codec support built in
+ */
+ public native static boolean supportsCompressionCodec(byte[] codec);
+
+ /**
* Config the native runtime with mapreduce job configurations.
*
* @param configs
Modified:
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc?rev=1616115&r1=1616114&r2=1616115&view=diff
==============================================================================
---
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
(original)
+++
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc
Wed Aug 6 07:32:49 2014
@@ -16,11 +16,15 @@
* limitations under the License.
*/
-#include "snappy-c.h"
+#include "config.h"
+
+#if defined HADOOP_SNAPPY_LIBRARY
#include "commons.h"
#include "NativeTask.h"
#include "SnappyCodec.h"
+#include <snappy-c.h>
+
namespace NativeTask {
SnappyCompressStream::SnappyCompressStream(OutputStream * stream, uint32_t
bufferSizeHint)
@@ -89,5 +93,6 @@ uint32_t SnappyDecompressStream::decompr
uint64_t SnappyDecompressStream::maxCompressedLength(uint64_t origLength) {
return snappy_max_compressed_length(origLength);
}
-
} // namespace NativeTask
+
+#endif // define HADOOP_SNAPPY_LIBRARY
Modified:
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc?rev=1616115&r1=1616114&r2=1616115&view=diff
==============================================================================
---
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
(original)
+++
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc
Wed Aug 6 07:32:49 2014
@@ -17,6 +17,7 @@
*/
#include "commons.h"
+#include "config.h"
#include "SyncUtils.h"
#include "Compressions.h"
#include "codec/GzipCodec.h"
@@ -110,7 +111,11 @@ CompressStream * Compressions::getCompre
return new GzipCompressStream(stream, bufferSizeHint);
}
if (codec == SnappyCodec.name) {
+#if defined HADOOP_SNAPPY_LIBRARY
return new SnappyCompressStream(stream, bufferSizeHint);
+#else
+ THROW_EXCEPTION(UnsupportException, "Snappy library is not loaded");
+#endif
}
if (codec == Lz4Codec.name) {
return new Lz4CompressStream(stream, bufferSizeHint);
@@ -124,7 +129,11 @@ DecompressStream * Compressions::getDeco
return new GzipDecompressStream(stream, bufferSizeHint);
}
if (codec == SnappyCodec.name) {
+#if defined HADOOP_SNAPPY_LIBRARY
return new SnappyDecompressStream(stream, bufferSizeHint);
+#else
+ THROW_EXCEPTION(UnsupportException, "Snappy library is not loaded");
+#endif
}
if (codec == Lz4Codec.name) {
return new Lz4DecompressStream(stream, bufferSizeHint);
Modified:
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc?rev=1616115&r1=1616114&r2=1616115&view=diff
==============================================================================
---
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
(original)
+++
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc
Wed Aug 6 07:32:49 2014
@@ -19,6 +19,7 @@
#ifndef QUICK_BUILD
#include "org_apache_hadoop_mapred_nativetask_NativeRuntime.h"
#endif
+#include "config.h"
#include "commons.h"
#include "jniutils.h"
#include "NativeObjectFactory.h"
@@ -31,6 +32,30 @@ using namespace NativeTask;
/*
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
+ * Method: supportCompressionCodec
+ * Signature: ([B)Z
+ */
+JNIEXPORT jboolean JNICALL
Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_supportsCompressionCodec
+ (JNIEnv *jenv, jclass clazz, jbyteArray codec)
+{
+ const std::string codecString = JNU_ByteArrayToString(jenv, codec);
+ if ("org.apache.hadoop.io.compress.GzipCodec" == codecString) {
+ return JNI_TRUE;
+ } else if ("org.apache.hadoop.io.compress.Lz4Codec" == codecString) {
+ return JNI_TRUE;
+ } else if ("org.apache.hadoop.io.compress.SnappyCodec" == codecString) {
+#if defined HADOOP_SNAPPY_LIBRARY
+ return JNI_TRUE;
+#else
+ return JNI_FALSE;
+#endif
+ } else {
+ return JNI_FALSE;
+ }
+}
+
+/*
+ * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIRelease
* Signature: ()V
*/JNIEXPORT void JNICALL
Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRelease(
Modified:
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h?rev=1616115&r1=1616114&r2=1616115&view=diff
==============================================================================
---
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h
(original)
+++
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/org_apache_hadoop_mapred_nativetask_NativeRuntime.h
Wed Aug 6 07:32:49 2014
@@ -9,6 +9,14 @@ extern "C" {
#endif
/*
* Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
+ * Method: supportCompressionCodec
+ * Signature: ([B)Z
+ */
+JNIEXPORT jboolean JNICALL
Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_supportsCompressionCodec
+ (JNIEnv *, jclass, jbyteArray);
+
+/*
+ * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime
* Method: JNIRelease
* Signature: ()V
*/JNIEXPORT void JNICALL
Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRelease(JNIEnv *,
Modified:
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc?rev=1616115&r1=1616114&r2=1616115&view=diff
==============================================================================
---
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
(original)
+++
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc
Wed Aug 6 07:32:49 2014
@@ -17,7 +17,7 @@
*/
#include "lz4.h"
-#include "snappy.h"
+#include "config.h"
#include "commons.h"
#include "Path.h"
#include "BufferStream.h"
@@ -25,9 +25,29 @@
#include "Compressions.h"
#include "test_commons.h"
-void TestCodec(const string & codec, const string & data, char * buff, char *
buff2, size_t buffLen,
- uint32_t buffhint) {
+#if defined HADOOP_SNAPPY_LIBRARY
+#include <snappy.h>
+#endif
+
+void TestCodec(const string & codec) {
+ string data;
+ size_t length = TestConfig.getInt("compression.input.length", 100 * 1024 *
1024);
+ uint32_t buffhint = TestConfig.getInt("compression.buffer.hint", 128 * 1024);
+ string type = TestConfig.get("compression.input.type", "bytes");
Timer timer;
+ GenerateKVTextLength(data, length, type);
+ LOG("%s", timer.getInterval("Generate data").c_str());
+
+ InputBuffer inputBuffer = InputBuffer(data);
+ size_t buffLen = data.length() / 2 * 3;
+
+ timer.reset();
+ char * buff = new char[buffLen];
+ char * buff2 = new char[buffLen];
+ memset(buff, 0, buffLen);
+ memset(buff2, 0, buffLen);
+ LOG("%s", timer.getInterval("memset buffer to prevent missing
page").c_str());
+
OutputBuffer outputBuffer = OutputBuffer(buff, buffLen);
CompressStream * compressor = Compressions::getCompressionStream(codec,
&outputBuffer, buffhint);
@@ -57,35 +77,10 @@ void TestCodec(const string & codec, con
ASSERT_EQ(data.length(), total);
ASSERT_EQ(0, memcmp(data.c_str(), buff2, total));
- delete compressor;
- delete decompressor;
-}
-
-TEST(Perf, Compressions) {
- string data;
- size_t length = TestConfig.getInt("compression.input.length", 100 * 1024 *
1024);
- uint32_t buffhint = TestConfig.getInt("compression.buffer.hint", 128 * 1024);
- string type = TestConfig.get("compression.input.type", "bytes");
- Timer timer;
- GenerateKVTextLength(data, length, type);
- LOG("%s", timer.getInterval("Generate data").c_str());
-
- InputBuffer inputBuffer = InputBuffer(data);
- size_t buffLen = data.length() / 2 * 3;
-
- timer.reset();
- char * buff = new char[buffLen];
- char * buff2 = new char[buffLen];
- memset(buff, 0, buffLen);
- memset(buff2, 0, buffLen);
- LOG("%s", timer.getInterval("memset buffer to prevent missing
page").c_str());
-
- TestCodec("org.apache.hadoop.io.compress.SnappyCodec", data, buff, buff2,
buffLen, buffhint);
- TestCodec("org.apache.hadoop.io.compress.Lz4Codec", data, buff, buff2,
buffLen, buffhint);
- TestCodec("org.apache.hadoop.io.compress.GzipCodec", data, buff, buff2,
buffLen, buffhint);
-
delete[] buff;
delete[] buff2;
+ delete compressor;
+ delete decompressor;
}
TEST(Perf, CompressionUtil) {
@@ -173,6 +168,9 @@ public:
}
};
+TEST(Perf, GzipCodec) {
+ TestCodec("org.apache.hadoop.io.compress.GzipCodec");
+}
void MeasureSingleFileLz4(const string & path, CompressResult & total, size_t
blockSize,
int times) {
@@ -226,6 +224,12 @@ TEST(Perf, RawCompressionLz4) {
printf("%s - Total\n", total.toString().c_str());
}
+TEST(Perf, Lz4Codec) {
+ TestCodec("org.apache.hadoop.io.compress.Lz4Codec");
+}
+
+#if defined HADOOP_SNAPPY_LIBRARY
+
void MeasureSingleFileSnappy(const string & path, CompressResult & total,
size_t blockSize,
int times) {
string data;
@@ -278,3 +282,8 @@ TEST(Perf, RawCompressionSnappy) {
printf("%s - Total\n", total.toString().c_str());
}
+TEST(Perf, SnappyCodec) {
+ TestCodec("org.apache.hadoop.io.compress.SnappyCodec");
+}
+
+#endif // define HADOOP_SNAPPY_LIBRARY
Modified:
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc?rev=1616115&r1=1616114&r2=1616115&view=diff
==============================================================================
---
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
(original)
+++
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc
Wed Aug 6 07:32:49 2014
@@ -18,6 +18,7 @@
#include <algorithm>
#include "commons.h"
+#include "config.h"
#include "BufferStream.h"
#include "FileSystem.h"
#include "IFile.h"
@@ -90,7 +91,9 @@ TEST(IFile, WriteRead) {
TestIFileReadWrite(TextType, partition, size, kvs);
TestIFileReadWrite(BytesType, partition, size, kvs);
TestIFileReadWrite(UnknownType, partition, size, kvs);
+#if defined HADOOP_SNAPPY_LIBRARY
TestIFileReadWrite(TextType, partition, size, kvs,
"org.apache.hadoop.io.compress.SnappyCodec");
+#endif
}
void TestIFileWriteRead2(vector<pair<string, string> > & kvs, char * buff,
size_t buffsize,
Modified:
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc?rev=1616115&r1=1616114&r2=1616115&view=diff
==============================================================================
---
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
(original)
+++
hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc
Wed Aug 6 07:32:49 2014
@@ -17,6 +17,7 @@
*/
#include "commons.h"
+#include "config.h"
#include "BufferStream.h"
#include "Buffers.h"
#include "test_commons.h"
@@ -43,6 +44,7 @@ TEST(Buffers, AppendRead) {
}
}
+#if defined HADOOP_SNAPPY_LIBRARY
TEST(Buffers, AppendReadSnappy) {
string codec = "org.apache.hadoop.io.compress.SnappyCodec";
vector<string> data;
@@ -64,4 +66,4 @@ TEST(Buffers, AppendReadSnappy) {
ASSERT_EQ(data[i], string(rd, data[i].length()));
}
}
-
+#endif
