MAPREDUCE-2841. Merge remote-tracking branch 'apache/MR-2841' into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a3a327c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a3a327c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a3a327c Branch: refs/heads/HDFS-6584 Commit: 8a3a327cd3c7fd1049b521325e485c59137855e8 Parents: a0ad975 b160707 Author: Todd Lipcon <t...@apache.org> Authored: Fri Sep 12 18:31:53 2014 -0700 Committer: Todd Lipcon <t...@apache.org> Committed: Fri Sep 12 18:31:53 2014 -0700 ---------------------------------------------------------------------- LICENSE.txt | 33 + .../assemblies/hadoop-mapreduce-dist.xml | 7 + .../CHANGES.MAPREDUCE-2841.txt | 26 + .../hadoop-mapreduce-client-nativetask/pom.xml | 216 + .../src/CMakeLists.txt | 276 + .../src/JNIFlags.cmake | 118 + .../src/config.h.cmake | 23 + .../hadoop/mapred/nativetask/Command.java | 57 + .../mapred/nativetask/CommandDispatcher.java | 33 + .../hadoop/mapred/nativetask/Constants.java | 48 + .../hadoop/mapred/nativetask/DataChannel.java | 41 + .../hadoop/mapred/nativetask/DataReceiver.java | 37 + .../mapred/nativetask/HadoopPlatform.java | 84 + .../mapred/nativetask/ICombineHandler.java | 44 + .../mapred/nativetask/INativeComparable.java | 54 + .../mapred/nativetask/INativeHandler.java | 53 + .../mapred/nativetask/NativeBatchProcessor.java | 279 + .../mapred/nativetask/NativeDataSource.java | 47 + .../mapred/nativetask/NativeDataTarget.java | 47 + .../NativeMapOutputCollectorDelegator.java | 171 + .../hadoop/mapred/nativetask/NativeRuntime.java | 197 + .../hadoop/mapred/nativetask/Platform.java | 100 + .../hadoop/mapred/nativetask/Platforms.java | 79 + .../mapred/nativetask/StatusReportChecker.java | 99 + .../hadoop/mapred/nativetask/TaskContext.java | 94 + .../mapred/nativetask/buffer/BufferType.java | 27 + .../nativetask/buffer/ByteBufferDataReader.java | 148 + .../nativetask/buffer/ByteBufferDataWriter.java | 169 + .../nativetask/buffer/DataInputStream.java | 27 + .../nativetask/buffer/DataOutputStream.java | 39 + .../mapred/nativetask/buffer/InputBuffer.java | 136 + .../mapred/nativetask/buffer/OutputBuffer.java | 76 + .../nativetask/handlers/BufferPullee.java | 121 + .../nativetask/handlers/BufferPuller.java | 203 + .../nativetask/handlers/BufferPushee.java | 151 + .../nativetask/handlers/BufferPusher.java | 91 + .../nativetask/handlers/CombinerHandler.java | 145 + .../mapred/nativetask/handlers/IDataLoader.java | 37 + .../handlers/NativeCollectorOnlyHandler.java | 171 + .../serde/BoolWritableSerializer.java | 35 + .../serde/ByteWritableSerializer.java | 35 + .../serde/BytesWritableSerializer.java | 48 + .../nativetask/serde/DefaultSerializer.java | 71 + .../serde/DoubleWritableSerializer.java | 35 + .../serde/FloatWritableSerializer.java | 36 + .../mapred/nativetask/serde/IKVSerializer.java | 48 + .../nativetask/serde/INativeSerializer.java | 49 + .../nativetask/serde/IntWritableSerializer.java | 35 + .../mapred/nativetask/serde/KVSerializer.java | 115 + .../serde/LongWritableSerializer.java | 34 + .../nativetask/serde/NativeSerialization.java | 91 + .../serde/NullWritableSerializer.java | 35 + .../serde/SerializationFramework.java | 35 + .../mapred/nativetask/serde/TextSerializer.java | 49 + .../serde/VIntWritableSerializer.java | 27 + .../serde/VLongWritableSerializer.java | 27 + .../mapred/nativetask/util/BytesUtil.java | 172 + .../mapred/nativetask/util/ConfigUtil.java | 47 + .../nativetask/util/LocalJobOutputFiles.java | 159 + .../nativetask/util/NativeTaskOutput.java | 106 + .../nativetask/util/NativeTaskOutputFiles.java | 170 + .../mapred/nativetask/util/OutputUtil.java | 47 + .../mapred/nativetask/util/ReadWriteBuffer.java | 159 + .../mapred/nativetask/util/SizedWritable.java | 55 + .../src/main/native/gtest/gtest-all.cc | 9120 ++++++++ .../src/main/native/gtest/gtest_main.cc | 39 + .../src/main/native/gtest/include/gtest/gtest.h | 19541 +++++++++++++++++ .../src/main/native/src/NativeTask.h | 507 + .../src/main/native/src/codec/BlockCodec.cc | 179 + .../src/main/native/src/codec/BlockCodec.h | 98 + .../src/main/native/src/codec/GzipCodec.cc | 192 + .../src/main/native/src/codec/GzipCodec.h | 82 + .../src/main/native/src/codec/Lz4Codec.cc | 88 + .../src/main/native/src/codec/Lz4Codec.h | 45 + .../src/main/native/src/codec/SnappyCodec.cc | 98 + .../src/main/native/src/codec/SnappyCodec.h | 46 + .../native/src/handler/AbstractMapHandler.cc | 34 + .../native/src/handler/AbstractMapHandler.h | 102 + .../src/main/native/src/handler/BatchHandler.cc | 355 + .../src/main/native/src/handler/BatchHandler.h | 157 + .../main/native/src/handler/CombineHandler.cc | 254 + .../main/native/src/handler/CombineHandler.h | 90 + .../src/handler/MCollectorOutputHandler.cc | 98 + .../src/handler/MCollectorOutputHandler.h | 53 + .../src/main/native/src/lib/BufferStream.cc | 43 + .../src/main/native/src/lib/BufferStream.h | 156 + .../src/main/native/src/lib/Buffers.cc | 235 + .../src/main/native/src/lib/Buffers.h | 637 + .../src/main/native/src/lib/Combiner.h | 70 + .../src/main/native/src/lib/Compressions.cc | 145 + .../src/main/native/src/lib/Compressions.h | 108 + .../src/main/native/src/lib/Constants.h | 26 + .../src/main/native/src/lib/FileSystem.cc | 263 + .../src/main/native/src/lib/FileSystem.h | 123 + .../src/main/native/src/lib/IFile.cc | 230 + .../src/main/native/src/lib/IFile.h | 166 + .../src/main/native/src/lib/Iterator.cc | 91 + .../src/main/native/src/lib/Iterator.h | 47 + .../src/main/native/src/lib/Log.cc | 30 + .../src/main/native/src/lib/Log.h | 48 + .../main/native/src/lib/MapOutputCollector.cc | 413 + .../main/native/src/lib/MapOutputCollector.h | 167 + .../src/main/native/src/lib/MapOutputSpec.cc | 64 + .../src/main/native/src/lib/MapOutputSpec.h | 81 + .../src/main/native/src/lib/MemoryBlock.cc | 67 + .../src/main/native/src/lib/MemoryBlock.h | 172 + .../src/main/native/src/lib/MemoryPool.h | 90 + .../src/main/native/src/lib/Merge.cc | 156 + .../src/main/native/src/lib/Merge.h | 254 + .../src/main/native/src/lib/MinHeap.h | 64 + .../src/main/native/src/lib/NativeLibrary.cc | 89 + .../src/main/native/src/lib/NativeLibrary.h | 57 + .../main/native/src/lib/NativeObjectFactory.cc | 444 + .../main/native/src/lib/NativeObjectFactory.h | 106 + .../main/native/src/lib/NativeRuntimeJniImpl.cc | 256 + .../src/main/native/src/lib/NativeTask.cc | 211 + .../src/main/native/src/lib/PartitionBucket.cc | 75 + .../src/main/native/src/lib/PartitionBucket.h | 142 + .../native/src/lib/PartitionBucketIterator.cc | 114 + .../native/src/lib/PartitionBucketIterator.h | 54 + .../src/main/native/src/lib/Path.cc | 53 + .../src/main/native/src/lib/Path.h | 38 + .../src/main/native/src/lib/SpillInfo.cc | 73 + .../src/main/native/src/lib/SpillInfo.h | 106 + .../main/native/src/lib/SpillOutputService.h | 44 + .../src/main/native/src/lib/Streams.cc | 122 + .../src/main/native/src/lib/Streams.h | 221 + .../src/main/native/src/lib/TaskCounters.cc | 40 + .../src/main/native/src/lib/TaskCounters.h | 44 + .../src/main/native/src/lib/commons.h | 52 + .../src/main/native/src/lib/jniutils.cc | 111 + .../src/main/native/src/lib/jniutils.h | 57 + .../src/main/native/src/lib/primitives.h | 287 + .../src/main/native/src/util/Checksum.cc | 749 + .../src/main/native/src/util/Checksum.h | 78 + .../main/native/src/util/DualPivotQuickSort.h | 137 + .../src/main/native/src/util/Random.cc | 285 + .../src/main/native/src/util/Random.h | 140 + .../src/main/native/src/util/StringUtil.cc | 213 + .../src/main/native/src/util/StringUtil.h | 66 + .../src/main/native/src/util/SyncUtils.cc | 55 + .../src/main/native/src/util/SyncUtils.h | 71 + .../src/main/native/src/util/Timer.cc | 116 + .../src/main/native/src/util/Timer.h | 56 + .../src/main/native/src/util/WritableUtils.cc | 309 + .../src/main/native/src/util/WritableUtils.h | 124 + .../src/main/native/test.sh | 36 + .../src/main/native/test/TestCommand.cc | 39 + .../src/main/native/test/TestCompressions.cc | 286 + .../src/main/native/test/TestConfig.cc | 67 + .../src/main/native/test/TestCounter.cc | 48 + .../src/main/native/test/TestFileSystem.cc | 50 + .../src/main/native/test/TestIFile.cc | 199 + .../src/main/native/test/TestMain.cc | 116 + .../src/main/native/test/TestPrimitives.cc | 304 + .../src/main/native/test/TestSort.cc | 306 + .../src/main/native/test/lib/TestByteArray.cc | 43 + .../src/main/native/test/lib/TestByteBuffer.cc | 47 + .../lib/TestComparatorForDualPivotQuickSort.cc | 89 + .../native/test/lib/TestComparatorForStdSort.cc | 89 + .../native/test/lib/TestFixSizeContainer.cc | 55 + .../src/main/native/test/lib/TestIterator.cc | 114 + .../src/main/native/test/lib/TestKVBuffer.cc | 54 + .../native/test/lib/TestMemBlockIterator.cc | 110 + .../src/main/native/test/lib/TestMemoryBlock.cc | 109 + .../src/main/native/test/lib/TestMemoryPool.cc | 48 + .../main/native/test/lib/TestPartitionBucket.cc | 222 + .../src/main/native/test/lib/TestReadBuffer.cc | 69 + .../main/native/test/lib/TestReadWriteBuffer.cc | 70 + .../src/main/native/test/test_commons.cc | 344 + .../src/main/native/test/test_commons.h | 150 + .../src/main/native/test/util/TestChecksum.cc | 43 + .../src/main/native/test/util/TestProcess.cc | 33 + .../src/main/native/test/util/TestStringUtil.cc | 64 + .../main/native/test/util/TestWritableUtils.cc | 49 + .../main/native/testData/testGlibCBugSpill.out | 2 + ...org.apache.hadoop.mapred.nativetask.Platform | 14 + .../mapred/nativetask/TestTaskContext.java | 44 + .../nativetask/buffer/TestBufferPushPull.java | 279 + .../buffer/TestByteBufferReadWrite.java | 168 + .../nativetask/buffer/TestInputBuffer.java | 53 + .../nativetask/buffer/TestOutputBuffer.java | 45 + .../nativetask/combinertest/CombinerTest.java | 124 + .../combinertest/LargeKVCombinerTest.java | 115 + .../combinertest/OldAPICombinerTest.java | 129 + .../nativetask/combinertest/WordCount.java | 91 + .../combinertest/WordCountWithOldAPI.java | 68 + .../nativetask/compresstest/CompressMapper.java | 63 + .../nativetask/compresstest/CompressTest.java | 144 + .../nativetask/handlers/TestCombineHandler.java | 80 + .../TestNativeCollectorOnlyHandler.java | 129 + .../nativetask/kvtest/HashSumReducer.java | 49 + .../hadoop/mapred/nativetask/kvtest/KVJob.java | 108 + .../hadoop/mapred/nativetask/kvtest/KVTest.java | 153 + .../mapred/nativetask/kvtest/LargeKVTest.java | 131 + .../mapred/nativetask/kvtest/TestInputFile.java | 227 + .../nativetask/nonsorttest/NonSortTest.java | 118 + .../nativetask/nonsorttest/NonSortTestMR.java | 72 + .../nativetask/serde/TestKVSerializer.java | 148 + .../serde/TestNativeSerialization.java | 114 + .../nativetask/testutil/BytesFactory.java | 121 + .../EnforceNativeOutputCollectorDelegator.java | 51 + .../nativetask/testutil/MockValueClass.java | 80 + .../nativetask/testutil/ResultVerifier.java | 162 + .../testutil/ScenarioConfiguration.java | 58 + .../nativetask/testutil/TestConstants.java | 71 + .../mapred/nativetask/testutil/TestInput.java | 61 + .../mapred/nativetask/utils/TestBytesUtil.java | 69 + .../nativetask/utils/TestReadWriteBuffer.java | 62 + .../nativetask/utils/TestSizedWritable.java | 34 + .../src/test/resources/common_conf.xml | 48 + .../src/test/resources/kvtest-conf.xml | 85 + .../src/test/resources/log4j.properties | 19 + .../src/test/resources/native_conf.xml | 30 + .../src/test/resources/normal_conf.xml | 26 + .../src/test/resources/test-combiner-conf.xml | 30 + .../src/test/resources/test-compress-conf.xml | 50 + .../src/test/resources/test-nonsort-conf.xml | 30 + .../hadoop-mapreduce-client/pom.xml | 1 + hadoop-mapreduce-project/pom.xml | 30 +- 220 files changed, 52881 insertions(+), 1 deletion(-) ----------------------------------------------------------------------
