Merge remote-tracking branch 'origin/trunk' into HDFS-6584
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ece3ecab Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ece3ecab Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ece3ecab Branch: refs/heads/HDFS-6581 Commit: ece3ecabdf348dd9f6876f2819e65f72c28cc25f Parents: 2689b6c 24d920b Author: Jing Zhao <ji...@apache.org> Authored: Mon Sep 15 10:45:45 2014 -0700 Committer: Jing Zhao <ji...@apache.org> Committed: Mon Sep 15 10:45:45 2014 -0700 ---------------------------------------------------------------------- LICENSE.txt | 33 + .../assemblies/hadoop-mapreduce-dist.xml | 7 + hadoop-common-project/hadoop-common/CHANGES.txt | 6 + .../src/main/conf/log4j.properties | 5 + .../src/main/resources/core-default.xml | 86 + .../apache/hadoop/crypto/key/TestKeyShell.java | 4 +- .../hadoop/security/alias/TestCredShell.java | 22 +- .../crypto/key/kms/server/KMSConfiguration.java | 15 +- .../hadoop/crypto/key/kms/server/MiniKMS.java | 3 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 3 +- hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 8 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../apache/hadoop/hdfs/TestEncryptionZones.java | 10 + hadoop-mapreduce-project/CHANGES.txt | 65 + .../mapreduce/v2/app/ControlledClock.java | 43 - .../v2/app/job/impl/TestTaskAttempt.java | 3 +- .../v2/app/rm/TestRMContainerAllocator.java | 3 +- .../mapreduce/v2/hs/TestHistoryFileManager.java | 3 +- .../hadoop-mapreduce-client-jobclient/pom.xml | 6 + .../v2/TestSpeculativeExecutionWithMRApp.java | 3 +- .../hadoop-mapreduce-client-nativetask/pom.xml | 216 + .../src/CMakeLists.txt | 276 + .../src/JNIFlags.cmake | 118 + .../src/config.h.cmake | 23 + .../hadoop/mapred/nativetask/Command.java | 57 + .../mapred/nativetask/CommandDispatcher.java | 33 + .../hadoop/mapred/nativetask/Constants.java | 48 + .../hadoop/mapred/nativetask/DataChannel.java | 41 + .../hadoop/mapred/nativetask/DataReceiver.java | 37 + .../mapred/nativetask/HadoopPlatform.java | 84 + .../mapred/nativetask/ICombineHandler.java | 44 + .../mapred/nativetask/INativeComparable.java | 54 + .../mapred/nativetask/INativeHandler.java | 53 + .../mapred/nativetask/NativeBatchProcessor.java | 279 + .../mapred/nativetask/NativeDataSource.java | 47 + .../mapred/nativetask/NativeDataTarget.java | 47 + .../NativeMapOutputCollectorDelegator.java | 171 + .../hadoop/mapred/nativetask/NativeRuntime.java | 197 + .../hadoop/mapred/nativetask/Platform.java | 100 + .../hadoop/mapred/nativetask/Platforms.java | 79 + .../mapred/nativetask/StatusReportChecker.java | 99 + .../hadoop/mapred/nativetask/TaskContext.java | 94 + .../mapred/nativetask/buffer/BufferType.java | 27 + .../nativetask/buffer/ByteBufferDataReader.java | 148 + .../nativetask/buffer/ByteBufferDataWriter.java | 169 + .../nativetask/buffer/DataInputStream.java | 27 + .../nativetask/buffer/DataOutputStream.java | 39 + .../mapred/nativetask/buffer/InputBuffer.java | 136 + .../mapred/nativetask/buffer/OutputBuffer.java | 76 + .../nativetask/handlers/BufferPullee.java | 121 + .../nativetask/handlers/BufferPuller.java | 203 + .../nativetask/handlers/BufferPushee.java | 151 + .../nativetask/handlers/BufferPusher.java | 91 + .../nativetask/handlers/CombinerHandler.java | 145 + .../mapred/nativetask/handlers/IDataLoader.java | 37 + .../handlers/NativeCollectorOnlyHandler.java | 171 + .../serde/BoolWritableSerializer.java | 35 + .../serde/ByteWritableSerializer.java | 35 + .../serde/BytesWritableSerializer.java | 48 + .../nativetask/serde/DefaultSerializer.java | 71 + .../serde/DoubleWritableSerializer.java | 35 + .../serde/FloatWritableSerializer.java | 36 + .../mapred/nativetask/serde/IKVSerializer.java | 48 + .../nativetask/serde/INativeSerializer.java | 49 + .../nativetask/serde/IntWritableSerializer.java | 35 + .../mapred/nativetask/serde/KVSerializer.java | 115 + .../serde/LongWritableSerializer.java | 34 + .../nativetask/serde/NativeSerialization.java | 91 + .../serde/NullWritableSerializer.java | 35 + .../serde/SerializationFramework.java | 35 + .../mapred/nativetask/serde/TextSerializer.java | 49 + .../serde/VIntWritableSerializer.java | 27 + .../serde/VLongWritableSerializer.java | 27 + .../mapred/nativetask/util/BytesUtil.java | 172 + .../mapred/nativetask/util/ConfigUtil.java | 47 + .../nativetask/util/LocalJobOutputFiles.java | 159 + .../nativetask/util/NativeTaskOutput.java | 106 + .../nativetask/util/NativeTaskOutputFiles.java | 170 + .../mapred/nativetask/util/OutputUtil.java | 47 + .../mapred/nativetask/util/ReadWriteBuffer.java | 159 + .../mapred/nativetask/util/SizedWritable.java | 55 + .../src/main/native/gtest/gtest-all.cc | 9120 ++++++++ .../src/main/native/gtest/gtest_main.cc | 39 + .../src/main/native/gtest/include/gtest/gtest.h | 19541 +++++++++++++++++ .../src/main/native/src/NativeTask.h | 507 + .../src/main/native/src/codec/BlockCodec.cc | 179 + .../src/main/native/src/codec/BlockCodec.h | 98 + .../src/main/native/src/codec/GzipCodec.cc | 192 + .../src/main/native/src/codec/GzipCodec.h | 82 + .../src/main/native/src/codec/Lz4Codec.cc | 88 + .../src/main/native/src/codec/Lz4Codec.h | 45 + .../src/main/native/src/codec/SnappyCodec.cc | 98 + .../src/main/native/src/codec/SnappyCodec.h | 46 + .../native/src/handler/AbstractMapHandler.cc | 34 + .../native/src/handler/AbstractMapHandler.h | 102 + .../src/main/native/src/handler/BatchHandler.cc | 355 + .../src/main/native/src/handler/BatchHandler.h | 157 + .../main/native/src/handler/CombineHandler.cc | 254 + .../main/native/src/handler/CombineHandler.h | 90 + .../src/handler/MCollectorOutputHandler.cc | 98 + .../src/handler/MCollectorOutputHandler.h | 53 + .../src/main/native/src/lib/BufferStream.cc | 43 + .../src/main/native/src/lib/BufferStream.h | 156 + .../src/main/native/src/lib/Buffers.cc | 235 + .../src/main/native/src/lib/Buffers.h | 637 + .../src/main/native/src/lib/Combiner.h | 70 + .../src/main/native/src/lib/Compressions.cc | 145 + .../src/main/native/src/lib/Compressions.h | 108 + .../src/main/native/src/lib/Constants.h | 26 + .../src/main/native/src/lib/FileSystem.cc | 263 + .../src/main/native/src/lib/FileSystem.h | 123 + .../src/main/native/src/lib/IFile.cc | 230 + .../src/main/native/src/lib/IFile.h | 166 + .../src/main/native/src/lib/Iterator.cc | 91 + .../src/main/native/src/lib/Iterator.h | 47 + .../src/main/native/src/lib/Log.cc | 30 + .../src/main/native/src/lib/Log.h | 48 + .../main/native/src/lib/MapOutputCollector.cc | 413 + .../main/native/src/lib/MapOutputCollector.h | 167 + .../src/main/native/src/lib/MapOutputSpec.cc | 64 + .../src/main/native/src/lib/MapOutputSpec.h | 81 + .../src/main/native/src/lib/MemoryBlock.cc | 67 + .../src/main/native/src/lib/MemoryBlock.h | 172 + .../src/main/native/src/lib/MemoryPool.h | 90 + .../src/main/native/src/lib/Merge.cc | 156 + .../src/main/native/src/lib/Merge.h | 254 + .../src/main/native/src/lib/MinHeap.h | 64 + .../src/main/native/src/lib/NativeLibrary.cc | 89 + .../src/main/native/src/lib/NativeLibrary.h | 57 + .../main/native/src/lib/NativeObjectFactory.cc | 444 + .../main/native/src/lib/NativeObjectFactory.h | 106 + .../main/native/src/lib/NativeRuntimeJniImpl.cc | 256 + .../src/main/native/src/lib/NativeTask.cc | 211 + .../src/main/native/src/lib/PartitionBucket.cc | 75 + .../src/main/native/src/lib/PartitionBucket.h | 142 + .../native/src/lib/PartitionBucketIterator.cc | 114 + .../native/src/lib/PartitionBucketIterator.h | 54 + .../src/main/native/src/lib/Path.cc | 53 + .../src/main/native/src/lib/Path.h | 38 + .../src/main/native/src/lib/SpillInfo.cc | 73 + .../src/main/native/src/lib/SpillInfo.h | 106 + .../main/native/src/lib/SpillOutputService.h | 44 + .../src/main/native/src/lib/Streams.cc | 122 + .../src/main/native/src/lib/Streams.h | 221 + .../src/main/native/src/lib/TaskCounters.cc | 40 + .../src/main/native/src/lib/TaskCounters.h | 44 + .../src/main/native/src/lib/commons.h | 52 + .../src/main/native/src/lib/jniutils.cc | 111 + .../src/main/native/src/lib/jniutils.h | 57 + .../src/main/native/src/lib/primitives.h | 287 + .../src/main/native/src/util/Checksum.cc | 749 + .../src/main/native/src/util/Checksum.h | 78 + .../main/native/src/util/DualPivotQuickSort.h | 137 + .../src/main/native/src/util/Random.cc | 285 + .../src/main/native/src/util/Random.h | 140 + .../src/main/native/src/util/StringUtil.cc | 213 + .../src/main/native/src/util/StringUtil.h | 66 + .../src/main/native/src/util/SyncUtils.cc | 55 + .../src/main/native/src/util/SyncUtils.h | 71 + .../src/main/native/src/util/Timer.cc | 116 + .../src/main/native/src/util/Timer.h | 56 + .../src/main/native/src/util/WritableUtils.cc | 309 + .../src/main/native/src/util/WritableUtils.h | 124 + .../src/main/native/test.sh | 36 + .../src/main/native/test/TestCommand.cc | 39 + .../src/main/native/test/TestCompressions.cc | 286 + .../src/main/native/test/TestConfig.cc | 67 + .../src/main/native/test/TestCounter.cc | 48 + .../src/main/native/test/TestFileSystem.cc | 50 + .../src/main/native/test/TestIFile.cc | 199 + .../src/main/native/test/TestMain.cc | 116 + .../src/main/native/test/TestPrimitives.cc | 304 + .../src/main/native/test/TestSort.cc | 306 + .../src/main/native/test/lib/TestByteArray.cc | 43 + .../src/main/native/test/lib/TestByteBuffer.cc | 47 + .../lib/TestComparatorForDualPivotQuickSort.cc | 89 + .../native/test/lib/TestComparatorForStdSort.cc | 89 + .../native/test/lib/TestFixSizeContainer.cc | 55 + .../src/main/native/test/lib/TestIterator.cc | 114 + .../src/main/native/test/lib/TestKVBuffer.cc | 54 + .../native/test/lib/TestMemBlockIterator.cc | 110 + .../src/main/native/test/lib/TestMemoryBlock.cc | 109 + .../src/main/native/test/lib/TestMemoryPool.cc | 48 + .../main/native/test/lib/TestPartitionBucket.cc | 222 + .../src/main/native/test/lib/TestReadBuffer.cc | 69 + .../main/native/test/lib/TestReadWriteBuffer.cc | 70 + .../src/main/native/test/test_commons.cc | 344 + .../src/main/native/test/test_commons.h | 150 + .../src/main/native/test/util/TestChecksum.cc | 43 + .../src/main/native/test/util/TestProcess.cc | 33 + .../src/main/native/test/util/TestStringUtil.cc | 64 + .../main/native/test/util/TestWritableUtils.cc | 49 + .../main/native/testData/testGlibCBugSpill.out | 2 + ...org.apache.hadoop.mapred.nativetask.Platform | 14 + .../mapred/nativetask/TestTaskContext.java | 44 + .../nativetask/buffer/TestBufferPushPull.java | 279 + .../buffer/TestByteBufferReadWrite.java | 168 + .../nativetask/buffer/TestInputBuffer.java | 53 + .../nativetask/buffer/TestOutputBuffer.java | 45 + .../nativetask/combinertest/CombinerTest.java | 124 + .../combinertest/LargeKVCombinerTest.java | 115 + .../combinertest/OldAPICombinerTest.java | 129 + .../nativetask/combinertest/WordCount.java | 91 + .../combinertest/WordCountWithOldAPI.java | 68 + .../nativetask/compresstest/CompressMapper.java | 63 + .../nativetask/compresstest/CompressTest.java | 144 + .../nativetask/handlers/TestCombineHandler.java | 80 + .../TestNativeCollectorOnlyHandler.java | 129 + .../nativetask/kvtest/HashSumReducer.java | 49 + .../hadoop/mapred/nativetask/kvtest/KVJob.java | 108 + .../hadoop/mapred/nativetask/kvtest/KVTest.java | 153 + .../mapred/nativetask/kvtest/LargeKVTest.java | 131 + .../mapred/nativetask/kvtest/TestInputFile.java | 227 + .../nativetask/nonsorttest/NonSortTest.java | 118 + .../nativetask/nonsorttest/NonSortTestMR.java | 72 + .../nativetask/serde/TestKVSerializer.java | 148 + .../serde/TestNativeSerialization.java | 114 + .../nativetask/testutil/BytesFactory.java | 121 + .../EnforceNativeOutputCollectorDelegator.java | 51 + .../nativetask/testutil/MockValueClass.java | 80 + .../nativetask/testutil/ResultVerifier.java | 162 + .../testutil/ScenarioConfiguration.java | 58 + .../nativetask/testutil/TestConstants.java | 71 + .../mapred/nativetask/testutil/TestInput.java | 61 + .../mapred/nativetask/utils/TestBytesUtil.java | 69 + .../nativetask/utils/TestReadWriteBuffer.java | 62 + .../nativetask/utils/TestSizedWritable.java | 34 + .../src/test/resources/common_conf.xml | 48 + .../src/test/resources/kvtest-conf.xml | 85 + .../src/test/resources/log4j.properties | 19 + .../src/test/resources/native_conf.xml | 30 + .../src/test/resources/normal_conf.xml | 26 + .../src/test/resources/test-combiner-conf.xml | 30 + .../src/test/resources/test-compress-conf.xml | 50 + .../src/test/resources/test-nonsort-conf.xml | 30 + .../hadoop-mapreduce-client/pom.xml | 1 + hadoop-mapreduce-project/pom.xml | 30 +- hadoop-project/pom.xml | 26 +- hadoop-tools/hadoop-aws/pom.xml | 10 + .../fs/s3a/AnonymousAWSCredentialsProvider.java | 37 + .../fs/s3a/BasicAWSCredentialsProvider.java | 51 + .../org/apache/hadoop/fs/s3a/Constants.java | 90 + .../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 62 + .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1019 + .../apache/hadoop/fs/s3a/S3AInputStream.java | 207 + .../apache/hadoop/fs/s3a/S3AOutputStream.java | 208 + .../services/org.apache.hadoop.fs.FileSystem | 1 + .../hadoop/fs/contract/s3a/S3AContract.java | 43 + .../fs/contract/s3a/TestS3AContractCreate.java | 38 + .../fs/contract/s3a/TestS3AContractDelete.java | 31 + .../fs/contract/s3a/TestS3AContractMkdir.java | 34 + .../fs/contract/s3a/TestS3AContractOpen.java | 31 + .../fs/contract/s3a/TestS3AContractRename.java | 64 + .../fs/contract/s3a/TestS3AContractRootDir.java | 35 + .../fs/contract/s3a/TestS3AContractSeek.java | 31 + .../fs/s3a/S3AFileSystemContractBaseTest.java | 327 + .../src/test/resources/contract/s3a.xml | 105 + .../src/test/resources/contract/s3n.xml | 7 +- hadoop-tools/hadoop-azure/pom.xml | 10 +- hadoop-yarn-project/CHANGES.txt | 21 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 2 +- .../records/ApplicationSubmissionContext.java | 43 + .../src/main/proto/yarn_protos.proto | 1 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 12 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 80 +- .../pb/ApplicationSubmissionContextPBImpl.java | 13 + .../hadoop/yarn/util/ControlledClock.java | 42 + .../timeline/webapp/CrossOriginFilter.java | 59 +- .../timeline/webapp/TestCrossOriginFilter.java | 86 +- .../recovery/FileSystemRMStateStore.java | 1 + .../recovery/MemoryRMStateStore.java | 1 + .../resourcemanager/recovery/RMStateStore.java | 13 +- .../recovery/ZKRMStateStore.java | 2 + .../records/ApplicationAttemptStateData.java | 15 +- .../pb/ApplicationAttemptStateDataPBImpl.java | 12 + .../server/resourcemanager/rmapp/RMAppImpl.java | 44 +- .../rmapp/attempt/RMAppAttempt.java | 6 + .../rmapp/attempt/RMAppAttemptImpl.java | 27 +- .../yarn_server_resourcemanager_recovery.proto | 1 + .../yarn/server/resourcemanager/MockRM.java | 23 +- .../server/resourcemanager/TestRMRestart.java | 43 + .../applicationsmanager/TestAMRestart.java | 127 + .../recovery/RMStateStoreTestBase.java | 6 +- 283 files changed, 56109 insertions(+), 185 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ece3ecab/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ----------------------------------------------------------------------
