MAPREDUCE-6069. Lint/style fixes and removal of unused code. Contributed by Todd Lipcon.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/683987be Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/683987be Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/683987be Branch: refs/heads/HDFS-6584 Commit: 683987be7c160e67ddb8534eeb3c464bbe2796dd Parents: 0032216 Author: Todd Lipcon <t...@apache.org> Authored: Wed Sep 3 13:07:24 2014 -0700 Committer: Todd Lipcon <t...@apache.org> Committed: Fri Sep 5 10:44:38 2014 -0700 ---------------------------------------------------------------------- .../CHANGES.MAPREDUCE-2841.txt | 1 + .../hadoop-mapreduce-client-nativetask/LICENSE | 22 -- .../src/CMakeLists.txt | 6 - .../hadoop/mapred/nativetask/Constants.java | 13 - .../hadoop/mapred/nativetask/DataReceiver.java | 3 - .../mapred/nativetask/ICombineHandler.java | 2 - .../mapred/nativetask/INativeHandler.java | 8 - .../mapred/nativetask/NativeBatchProcessor.java | 11 - .../NativeMapOutputCollectorDelegator.java | 10 +- .../hadoop/mapred/nativetask/NativeRuntime.java | 32 +- .../hadoop/mapred/nativetask/Platform.java | 11 +- .../nativetask/handlers/BufferPullee.java | 3 +- .../nativetask/handlers/BufferPuller.java | 2 - .../nativetask/handlers/BufferPushee.java | 6 +- .../nativetask/handlers/BufferPusher.java | 3 +- .../nativetask/handlers/CombinerHandler.java | 16 +- .../mapred/nativetask/handlers/IDataLoader.java | 1 - .../handlers/NativeCollectorOnlyHandler.java | 6 +- .../serde/BytesWritableSerializer.java | 3 +- .../mapred/nativetask/serde/IKVSerializer.java | 31 +- .../mapred/nativetask/serde/KVSerializer.java | 6 +- .../nativetask/serde/NativeSerialization.java | 11 +- .../mapred/nativetask/util/BytesUtil.java | 2 +- .../nativetask/util/LocalJobOutputFiles.java | 58 +--- .../nativetask/util/NativeTaskOutput.java | 55 +--- .../nativetask/util/NativeTaskOutputFiles.java | 72 ++--- .../mapred/nativetask/util/ReadWriteBuffer.java | 16 +- .../src/main/native/cityhash/city.cc | 307 ------------------- .../src/main/native/cityhash/city.h | 90 ------ .../src/main/native/src/NativeTask.h | 137 --------- .../src/main/native/src/codec/BlockCodec.cc | 5 +- .../src/main/native/src/codec/BlockCodec.h | 2 +- .../src/main/native/src/codec/GzipCodec.cc | 2 +- .../src/main/native/src/codec/GzipCodec.h | 2 +- .../src/main/native/src/codec/Lz4Codec.cc | 2 +- .../src/main/native/src/codec/Lz4Codec.h | 2 +- .../src/main/native/src/codec/SnappyCodec.cc | 2 +- .../src/main/native/src/codec/SnappyCodec.h | 2 +- .../native/src/handler/AbstractMapHandler.cc | 9 +- .../src/main/native/src/handler/BatchHandler.cc | 9 +- .../main/native/src/handler/CombineHandler.cc | 4 +- .../main/native/src/handler/CombineHandler.h | 2 +- .../src/handler/MCollectorOutputHandler.cc | 6 +- .../src/main/native/src/lib/BufferStream.cc | 116 +------ .../src/main/native/src/lib/BufferStream.h | 39 +-- .../src/main/native/src/lib/Buffers.cc | 77 +---- .../src/main/native/src/lib/Buffers.h | 63 +--- .../src/main/native/src/lib/Combiner.cc | 73 ----- .../src/main/native/src/lib/Combiner.h | 18 +- .../src/main/native/src/lib/Compressions.cc | 6 +- .../src/main/native/src/lib/Compressions.h | 2 +- .../src/main/native/src/lib/FileSystem.cc | 23 +- .../src/main/native/src/lib/FileSystem.h | 2 +- .../src/main/native/src/lib/IFile.cc | 6 +- .../src/main/native/src/lib/IFile.h | 10 +- .../src/main/native/src/lib/Iterator.cc | 6 +- .../src/main/native/src/lib/Log.cc | 2 +- .../main/native/src/lib/MapOutputCollector.cc | 58 ++-- .../main/native/src/lib/MapOutputCollector.h | 16 +- .../src/main/native/src/lib/MapOutputSpec.cc | 4 +- .../src/main/native/src/lib/MapOutputSpec.h | 4 +- .../src/main/native/src/lib/MemoryBlock.cc | 24 +- .../src/main/native/src/lib/MemoryPool.h | 4 +- .../src/main/native/src/lib/Merge.cc | 5 +- .../src/main/native/src/lib/Merge.h | 8 +- .../src/main/native/src/lib/MinHeap.h | 2 +- .../src/main/native/src/lib/NativeLibrary.cc | 7 +- .../main/native/src/lib/NativeObjectFactory.cc | 19 +- .../main/native/src/lib/NativeRuntimeJniImpl.cc | 27 +- .../src/main/native/src/lib/NativeTask.cc | 44 +-- .../src/main/native/src/lib/PartitionBucket.cc | 25 +- .../src/main/native/src/lib/PartitionBucket.h | 16 +- .../native/src/lib/PartitionBucketIterator.cc | 22 +- .../native/src/lib/PartitionBucketIterator.h | 16 +- .../src/main/native/src/lib/Path.cc | 2 +- .../src/main/native/src/lib/SpillInfo.cc | 12 +- .../src/main/native/src/lib/Streams.cc | 6 +- .../src/main/native/src/lib/TaskCounters.cc | 3 +- .../src/main/native/src/lib/TrackingCollector.h | 50 --- .../src/main/native/src/lib/commons.h | 9 +- .../src/main/native/src/lib/jniutils.cc | 8 +- .../src/main/native/src/util/Checksum.cc | 2 +- .../src/main/native/src/util/Hash.cc | 24 -- .../src/main/native/src/util/Hash.h | 73 ----- .../src/main/native/src/util/Random.cc | 4 +- .../src/main/native/src/util/StringUtil.cc | 9 +- .../src/main/native/src/util/SyncUtils.cc | 114 +------ .../src/main/native/src/util/SyncUtils.h | 228 -------------- .../src/main/native/src/util/Timer.cc | 7 +- .../src/main/native/src/util/WritableUtils.cc | 6 +- .../src/main/native/src/util/WritableUtils.h | 2 +- .../src/main/native/test/TestCommand.cc | 9 +- .../src/main/native/test/TestCompressions.cc | 14 +- .../src/main/native/test/TestConfig.cc | 6 +- .../src/main/native/test/TestCounter.cc | 8 +- .../src/main/native/test/TestFileSystem.cc | 2 +- .../src/main/native/test/TestIFile.cc | 15 +- .../src/main/native/test/TestMain.cc | 21 +- .../src/main/native/test/TestPrimitives.cc | 2 +- .../src/main/native/test/TestSort.cc | 33 +- .../src/main/native/test/lib/TestByteArray.cc | 2 +- .../src/main/native/test/lib/TestByteBuffer.cc | 2 +- .../lib/TestComparatorForDualPivotQuickSort.cc | 2 +- .../native/test/lib/TestComparatorForStdSort.cc | 2 +- .../native/test/lib/TestFixSizeContainer.cc | 2 +- .../src/main/native/test/lib/TestIterator.cc | 4 +- .../src/main/native/test/lib/TestKVBuffer.cc | 2 +- .../native/test/lib/TestMemBlockIterator.cc | 11 +- .../src/main/native/test/lib/TestMemoryBlock.cc | 8 +- .../src/main/native/test/lib/TestMemoryPool.cc | 14 +- .../main/native/test/lib/TestPartitionBucket.cc | 36 +-- .../src/main/native/test/lib/TestReadBuffer.cc | 6 +- .../main/native/test/lib/TestReadWriteBuffer.cc | 2 +- .../native/test/lib/TestTrackingCollector.cc | 37 --- .../src/main/native/test/test_commons.cc | 4 +- .../src/main/native/test/test_commons.h | 4 +- .../src/main/native/test/util/TestHash.cc | 52 ---- .../src/main/native/test/util/TestSyncUtils.cc | 116 ------- .../nativetask/combinertest/CombinerTest.java | 9 +- .../combinertest/LargeKVCombinerTest.java | 11 +- .../combinertest/OldAPICombinerTest.java | 21 +- .../nativetask/combinertest/WordCount.java | 3 +- .../combinertest/WordCountWithOldAPI.java | 9 +- .../nativetask/compresstest/CompressMapper.java | 6 +- .../nativetask/handlers/TestCombineHandler.java | 7 +- .../TestNativeCollectorOnlyHandler.java | 15 +- .../nativetask/kvtest/HashSumReducer.java | 3 +- .../hadoop/mapred/nativetask/kvtest/KVJob.java | 17 +- .../mapred/nativetask/kvtest/TestInputFile.java | 3 +- .../nativetask/nonsorttest/NonSortTest.java | 14 +- .../nativetask/nonsorttest/NonSortTestMR.java | 3 +- .../nativetask/serde/TestKVSerializer.java | 18 +- .../serde/TestNativeSerialization.java | 3 +- .../EnforceNativeOutputCollectorDelegator.java | 3 +- .../nativetask/testutil/ResultVerifier.java | 15 +- .../mapred/nativetask/testutil/TestInput.java | 9 +- 136 files changed, 546 insertions(+), 2335 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt index 279b960..ea17907 100644 --- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt +++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt @@ -21,3 +21,4 @@ MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is suces MAPREDUCE-6056. native-task: move system test working dir to target dir and cleanup test config xml files (Manu Zhang via bchang) MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cleanup (todd) MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang) +MAPREDUCE-6069. native-task: Lint/style fixes and removal of unused code (todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE index 612fdf9..0780741 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE @@ -204,28 +204,6 @@ Licenses for third party projects used by this project: -CityHash src/main/native/cityhash ---------------------------------------------------------------------- -// Copyright (c) 2011 Google, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - GoogleTest src/main/native/gtest --------------------------------------------------------------------- Copyright 2008, Google Inc. http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt index 0ab99db..7eed176 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt @@ -178,7 +178,6 @@ configure_file(main/native/test.sh test/test.sh) add_dual_library(nativetask ${CMAKE_BINARY_DIR}/lz4.c - ${D}/cityhash/city.cc ${D}/src/codec/BlockCodec.cc ${D}/src/codec/GzipCodec.cc ${D}/src/codec/Lz4Codec.cc @@ -208,10 +207,8 @@ add_dual_library(nativetask ${D}/src/lib/SpillInfo.cc ${D}/src/lib/Path.cc ${D}/src/lib/Streams.cc - ${D}/src/lib/Combiner.cc ${D}/src/lib/TaskCounters.cc ${D}/src/util/Checksum.cc - ${D}/src/util/Hash.cc ${D}/src/util/Random.cc ${D}/src/util/StringUtil.cc ${D}/src/util/SyncUtils.cc @@ -237,11 +234,8 @@ add_executable(nttest ${D}/test/lib/TestPartitionBucket.cc ${D}/test/lib/TestReadBuffer.cc ${D}/test/lib/TestReadWriteBuffer.cc - ${D}/test/lib/TestTrackingCollector.cc ${D}/test/util/TestChecksum.cc - ${D}/test/util/TestHash.cc ${D}/test/util/TestStringUtil.cc - ${D}/test/util/TestSyncUtils.cc ${D}/test/util/TestWritableUtils.cc ${D}/test/TestCommand.cc ${D}/test/TestConfig.cc http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java index ff1a95c..1f7a5d1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java @@ -25,22 +25,9 @@ public class Constants { public static final String MAP_SORT_CLASS = "map.sort.class"; public static final String MAPRED_COMBINER_CLASS = "mapred.combiner.class"; - public static final String MAPRED_MAPTASK_DELEGATOR_CLASS = "mapreduce.map.task.delegator.class"; - public static final String MAPRED_REDUCETASK_DELEGATOR_CLASS = "mapreduce.reduce.task.delegator.class"; - public static final String NATIVE_TASK_ENABLED = "native.task.enabled"; public static final String NATIVE_LOG_DEVICE = "native.log.device"; public static final String NATIVE_HADOOP_VERSION = "native.hadoop.version"; - public static final String NATIVE_MAPPER_CLASS = "native.mapper.class"; - public static final String NATIVE_REDUCER_CLASS = "native.reducer.class"; - public static final String NATIVE_PARTITIONER_CLASS = "native.partitioner.class"; - public static final String NATIVE_COMBINER_CLASS = "native.combiner.class"; - public static final String NATIVE_INPUT_SPLIT = "native.input.split"; - - public static final String NATIVE_RECORDREADER_CLASS = "native.recordreader.class"; - public static final String NATIVE_RECORDWRITER_CLASS = "native.recordwriter.class"; - public static final String NATIVE_OUTPUT_FILE_NAME = "native.output.file.name"; - public static final String NATIVE_PROCESSOR_BUFFER_KB = "native.processor.buffer.kb"; public static final int NATIVE_PROCESSOR_BUFFER_KB_DEFAULT = 64; public static final int NATIVE_ASYNC_PROCESSOR_BUFFER_KB_DEFAULT = 1024; http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java index 85f78ca..b3bdf53 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/DataReceiver.java @@ -32,9 +32,6 @@ public interface DataReceiver { /** * Send a signal to the receiver that the data arrives. * The data is transferred in another band. - * - * @return - * @throws IOException */ public boolean receiveData() throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java index e4e706b..d28b883c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/ICombineHandler.java @@ -29,7 +29,6 @@ public interface ICombineHandler { /** * run combiner - * @throws IOException */ public void combine() throws IOException; @@ -40,7 +39,6 @@ public interface ICombineHandler { /** * close handlers, buffer pullers and pushers - * @throws IOException */ public void close() throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java index 9506c7f..fef56d9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/INativeHandler.java @@ -45,17 +45,9 @@ public interface INativeHandler extends NativeDataTarget, NativeDataSource { /** * call command to downstream - * - * @param command - * @param parameter - * @return - * @throws IOException */ public ReadWriteBuffer call(Command command, ReadWriteBuffer parameter) throws IOException; - /** - * @param handler - */ void setCommandDispatcher(CommandDispatcher handler); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java index 3c3811f..9458c6b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java @@ -229,32 +229,21 @@ public class NativeBatchProcessor implements INativeHandler { /** * Let native side to process data in inputBuffer - * - * @param handler - * @param length */ private native void nativeProcessInput(long handler, int length); /** * Notice native side input is finished - * - * @param handler */ private native void nativeFinish(long handler); /** * Send control message to native side - * - * @param cmd - * command data - * @return return value */ private native byte[] nativeCommand(long handler, int cmd, byte[] parameter); /** * Load data from native - * - * @return */ private native void nativeLoadData(long handler); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java index 828d7df..5a516c4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java @@ -95,7 +95,8 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect if (!QuickSort.class.getName().equals(job.get(Constants.MAP_SORT_CLASS))) { - String message = "Native-Task doesn't support sort class " + job.get(Constants.MAP_SORT_CLASS); + String message = "Native-Task doesn't support sort class " + + job.get(Constants.MAP_SORT_CLASS); LOG.error(message); throw new InvalidJobConfException(message); } @@ -115,8 +116,8 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect LOG.error(message); throw new InvalidJobConfException(message); } else if (!Platforms.support(keyCls.getName(), serializer, job)) { - String message = "Native output collector doesn't support this key, this key is not comparable in native " - + keyCls.getName(); + String message = "Native output collector doesn't support this key, " + + "this key is not comparable in native: " + keyCls.getName(); LOG.error(message); throw new InvalidJobConfException(message); } @@ -144,7 +145,8 @@ public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollect updater.start(); } else { - String message = "Nativeruntime cannot be loaded, please check the libnativetask.so is in hadoop library dir"; + String message = "NativeRuntime cannot be loaded, please check that " + + "libnativetask.so is in hadoop library dir"; LOG.error(message); throw new InvalidJobConfException(message); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java index 395cc89..a0e88bd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java @@ -79,9 +79,6 @@ public class NativeRuntime { /** * create native object We use it to create native handlers - * - * @param clazz - * @return */ public synchronized static long createNativeObject(String clazz) { assertNativeLibraryLoaded(); @@ -94,13 +91,11 @@ public class NativeRuntime { /** * Register a customized library - * - * @param clazz - * @return */ public synchronized static long registerLibrary(String libraryName, String clazz) { assertNativeLibraryLoaded(); - final long ret = JNIRegisterModule(libraryName.getBytes(Charsets.UTF_8), clazz.getBytes(Charsets.UTF_8)); + final long ret = JNIRegisterModule(libraryName.getBytes(Charsets.UTF_8), + clazz.getBytes(Charsets.UTF_8)); if (ret != 0) { LOG.warn("Can't create NativeObject for class " + clazz + ", probably not exist."); } @@ -117,9 +112,6 @@ public class NativeRuntime { /** * Get the status report from native space - * - * @param reporter - * @throws IOException */ public static void reportStatus(TaskReporter reporter) throws IOException { assertNativeLibraryLoaded(); @@ -164,40 +156,32 @@ public class NativeRuntime { /** * Config the native runtime with mapreduce job configurations. - * - * @param configs */ private native static void JNIConfigure(byte[][] configs); /** * create a native object in native space - * - * @param clazz - * @return */ private native static long JNICreateNativeObject(byte[] clazz); /** * create the default native object for certain type - * - * @param type - * @return */ @Deprecated private native static long JNICreateDefaultNativeObject(byte[] type); /** * destroy native object in native space - * - * @param addr */ private native static void JNIReleaseNativeObject(long addr); /** - * get status update from native side Encoding: progress:float status:Text Counter number: int the count of the - * counters Counters: array [group:Text, name:Text, incrCount:Long] - * - * @return + * Get status update from native side + * Encoding: + * progress:float + * status:Text + * number: int the count of the counters + * Counters: array [group:Text, name:Text, incrCount:Long] */ private native static byte[] JNIUpdateStatus(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java index 7e41989..efec26c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platform.java @@ -49,8 +49,6 @@ public abstract class Platform { /** * initialize a platform, where we should call registerKey - * - * @throws IOException */ public abstract void init() throws IOException; @@ -65,7 +63,6 @@ public abstract class Platform { * * @param keyClassName map out key class name * @param key key serializer class - * @throws IOException */ protected void registerKey(String keyClassName, Class<?> key) throws IOException { serialization.register(keyClassName, key); @@ -92,12 +89,12 @@ public abstract class Platform { /** * whether it's the platform that has defined a custom Java comparator * - * NativeTask doesn't support custom Java comparator(set with mapreduce.job.output.key.comparator.class) - * but a platform (e.g Pig) could also set that conf and implement native comparators so - * we shouldn't bail out. + * NativeTask doesn't support custom Java comparators + * (set with mapreduce.job.output.key.comparator.class) + * but a platform (e.g Pig) could also set that conf and implement native + * comparators so we shouldn't bail out. * * @param keyComparator comparator set with mapreduce.job.output.key.comparator.class - * @return */ protected abstract boolean define(Class<?> keyComparator); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java index 1047bbf..9fd7a04 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPullee.java @@ -46,7 +46,8 @@ public class BufferPullee<IK, IV> implements IDataLoader { private final NativeDataTarget target; private boolean closed = false; - public BufferPullee(Class<IK> iKClass, Class<IV> iVClass, RawKeyValueIterator rIter, NativeDataTarget target) + public BufferPullee(Class<IK> iKClass, Class<IV> iVClass, + RawKeyValueIterator rIter, NativeDataTarget target) throws IOException { this.rIter = rIter; tmpInputKey = new SizedWritable<IK>(iKClass); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java index 18523ef..b5dff92 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java @@ -189,8 +189,6 @@ public class BufferPuller implements RawKeyValueIterator, DataReceiver { /** * Closes the iterator so that the underlying streams can be closed. - * - * @throws IOException */ @Override public void close() throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java index fe78325..090882d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java @@ -51,7 +51,8 @@ public class BufferPushee<OK, OV> implements Closeable { private KVSerializer<OK, OV> deserializer; private boolean closed = false; - public BufferPushee(Class<OK> oKClass, Class<OV> oVClass, RecordWriter<OK, OV> writer) throws IOException { + public BufferPushee(Class<OK> oKClass, Class<OV> oVClass, + RecordWriter<OK, OV> writer) throws IOException { tmpOutputKey = new SizedWritable<OK>(oKClass); tmpOutputValue = new SizedWritable<OV>(oVClass); @@ -128,7 +129,8 @@ public class BufferPushee<OK, OV> implements Closeable { } } if (remain != totalRead) { - throw new IOException("We expect to read " + remain + ", but we actually read: " + totalRead); + throw new IOException("We expect to read " + remain + + ", but we actually read: " + totalRead); } return true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java index 10d619b..454127c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java @@ -44,7 +44,8 @@ public class BufferPusher<K, V> implements OutputCollector<K, V> { IKVSerializer serializer; private boolean closed = false; - public BufferPusher(Class<K> iKClass, Class<V> iVClass, NativeDataTarget target) throws IOException { + public BufferPusher(Class<K> iKClass, Class<V> iVClass, + NativeDataTarget target) throws IOException { tmpInputKey = new SizedWritable<K>(iKClass); tmpInputValue = new SizedWritable<V>(iVClass); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java index dad7af9..fcaeca1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java @@ -49,7 +49,8 @@ class CombinerHandler<K, V> implements ICombineHandler, CommandDispatcher { private final BufferPusher<K, V> kvPusher; private boolean closed = false; - public static <K, V> ICombineHandler create(TaskContext context) throws IOException, ClassNotFoundException { + public static <K, V> ICombineHandler create(TaskContext context) + throws IOException, ClassNotFoundException { final JobConf conf = new JobConf(context.getConf()); conf.set(Constants.SERIALIZATION_FRAMEWORK, String.valueOf(SerializationFramework.WRITABLE_SERIALIZATION.getType())); @@ -66,11 +67,13 @@ class CombinerHandler<K, V> implements ICombineHandler, CommandDispatcher { final Counter combineInputCounter = context.getTaskReporter().getCounter( TaskCounter.COMBINE_INPUT_RECORDS); - - final CombinerRunner<K, V> combinerRunner = CombinerRunner.create(conf, context.getTaskAttemptId(), + + final CombinerRunner<K, V> combinerRunner = CombinerRunner.create( + conf, context.getTaskAttemptId(), combineInputCounter, context.getTaskReporter(), null); - final INativeHandler nativeHandler = NativeBatchProcessor.create(NAME, conf, DataChannel.INOUT); + final INativeHandler nativeHandler = NativeBatchProcessor.create( + NAME, conf, DataChannel.INOUT); @SuppressWarnings("unchecked") final BufferPusher<K, V> pusher = new BufferPusher<K, V>((Class<K>)context.getInputKeyClass(), (Class<V>)context.getInputValueClass(), @@ -79,8 +82,9 @@ class CombinerHandler<K, V> implements ICombineHandler, CommandDispatcher { return new CombinerHandler<K, V>(nativeHandler, combinerRunner, puller, pusher); } - public CombinerHandler(INativeHandler nativeHandler, CombinerRunner<K, V> combiner, BufferPuller puller, - BufferPusher<K, V> kvPusher) throws IOException { + public CombinerHandler(INativeHandler nativeHandler, CombinerRunner<K, V> combiner, + BufferPuller puller, BufferPusher<K, V> kvPusher) + throws IOException { this.nativeHandler = nativeHandler; this.combinerRunner = combiner; this.puller = puller; http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java index d2cbf1f..6eaf28b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/IDataLoader.java @@ -30,7 +30,6 @@ public interface IDataLoader { /** * @return size of data loaded - * @throws IOException */ public int load() throws IOException; http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java index 284c3dc..34238ae 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java @@ -63,7 +63,8 @@ public class NativeCollectorOnlyHandler<K, V> implements CommandDispatcher, Clos private final INativeHandler nativeHandler; private boolean closed = false; - public static <K, V> NativeCollectorOnlyHandler<K, V> create(TaskContext context) throws IOException { + public static <K, V> NativeCollectorOnlyHandler<K, V> create(TaskContext context) + throws IOException { ICombineHandler combinerHandler = null; @@ -81,7 +82,8 @@ public class NativeCollectorOnlyHandler<K, V> implements CommandDispatcher, Clos LOG.info("[NativeCollectorOnlyHandler] combiner is not null"); } - final INativeHandler nativeHandler = NativeBatchProcessor.create(NAME, context.getConf(), DataChannel.OUT); + final INativeHandler nativeHandler = NativeBatchProcessor.create( + NAME, context.getConf(), DataChannel.OUT); final BufferPusher<K, V> kvPusher = new BufferPusher<K, V>( (Class<K>)context.getOutputKeyClass(), (Class<V>)context.getOutputValueClass(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java index 08264f9..48dccfa 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/BytesWritableSerializer.java @@ -27,7 +27,8 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.mapred.nativetask.INativeComparable; @InterfaceAudience.Private -public class BytesWritableSerializer implements INativeComparable, INativeSerializer<BytesWritable> { +public class BytesWritableSerializer + implements INativeComparable, INativeSerializer<BytesWritable> { @Override public int getLength(BytesWritable w) throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java index 8506d17..15a2ef7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/IKVSerializer.java @@ -33,43 +33,16 @@ public interface IKVSerializer { /** * update the length field of SizedWritable - * @param key - * @param value - * @throws IOException */ public void updateLength(SizedWritable<?> key, SizedWritable<?> value) throws IOException; - /** - * - * @param out - * @param key - * @param value - * @return bytes written - * @throws IOException - */ public int serializeKV(DataOutputStream out, SizedWritable<?> key, SizedWritable<?> value) throws IOException; - /** - * serialize partitionId as well - * @param out - * @param partitionId - * @param key - * @param value - * @return - * @throws IOException - */ public int serializePartitionKV(DataOutputStream out, int partitionId, SizedWritable<?> key, SizedWritable<?> value) throws IOException; - /** - * - * @param in - * @param key - * @param value - * @return bytes read - * @throws IOException - */ - public int deserializeKV(DataInputStream in, SizedWritable<?> key, SizedWritable<?> value) throws IOException; + public int deserializeKV(DataInputStream in, SizedWritable<?> key, SizedWritable<?> value) + throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java index d650a4d..e6609b8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java @@ -55,7 +55,8 @@ public class KVSerializer<K, V> implements IKVSerializer { } @Override - public int serializeKV(DataOutputStream out, SizedWritable<?> key, SizedWritable<?> value) throws IOException { + public int serializeKV(DataOutputStream out, SizedWritable<?> key, SizedWritable<?> value) + throws IOException { return serializePartitionKV(out, -1, key, value); } @@ -64,7 +65,8 @@ public class KVSerializer<K, V> implements IKVSerializer { SizedWritable<?> key, SizedWritable<?> value) throws IOException { - if (key.length == SizedWritable.INVALID_LENGTH || value.length == SizedWritable.INVALID_LENGTH) { + if (key.length == SizedWritable.INVALID_LENGTH || + value.length == SizedWritable.INVALID_LENGTH) { updateLength(key, value); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java index 5881a46..11bde64 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java @@ -27,7 +27,8 @@ import org.apache.hadoop.io.Writable; @InterfaceAudience.Private public class NativeSerialization { - private final ConcurrentHashMap<String, Class<?>> map = new ConcurrentHashMap<String, Class<?>>(); + private final ConcurrentHashMap<String, Class<?>> map = + new ConcurrentHashMap<String, Class<?>>(); public boolean accept(Class<?> c) { return Writable.class.isAssignableFrom(c); @@ -40,7 +41,8 @@ public class NativeSerialization { return null; } if (!Writable.class.isAssignableFrom(c)) { - throw new IOException("Cannot serialize type " + c.getName() + ", we only accept subclass of Writable"); + throw new IOException("Cannot serialize type " + c.getName() + + ", we only accept subclass of Writable"); } final String name = c.getName(); final Class<?> serializer = map.get(name); @@ -70,8 +72,9 @@ public class NativeSerialization { return; } else { if (!storedSerializer.getName().equals(serializer.getName())) { - throw new IOException("Error! Serializer already registered, exist: " + storedSerializer.getName() + ", new: " - + serializer.getName()); + throw new IOException("Error! Serializer already registered, existing: " + + storedSerializer.getName() + ", new: " + + serializer.getName()); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java index 35ef0a9..16f6919 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java @@ -97,7 +97,7 @@ public class BytesUtil { * Write a printable representation of a byte array. * * @param b byte array - * @return string + * @return the printable presentation * @see #toStringBinary(byte[], int, int) */ public static String toStringBinary(final byte [] b) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java index 6d63be5..aa97789 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/LocalJobOutputFiles.java @@ -45,9 +45,6 @@ public class LocalJobOutputFiles implements NativeTaskOutput { /** * Return the path to local map output file created earlier - * - * @return path - * @throws IOException */ public Path getOutputFile() throws IOException { String path = String.format(OUTPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT); @@ -57,10 +54,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput { /** * Create a local map output file name. * - * @param size - * the size of the file - * @return path - * @throws IOException + * @param size the size of the file */ public Path getOutputFileForWrite(long size) throws IOException { String path = String.format(OUTPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT); @@ -69,9 +63,6 @@ public class LocalJobOutputFiles implements NativeTaskOutput { /** * Return the path to a local map output index file created earlier - * - * @return path - * @throws IOException */ public Path getOutputIndexFile() throws IOException { String path = String.format(OUTPUT_FILE_INDEX_FORMAT_STRING, TASKTRACKER_OUTPUT); @@ -81,10 +72,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput { /** * Create a local map output index file name. * - * @param size - * the size of the file - * @return path - * @throws IOException + * @param size the size of the file */ public Path getOutputIndexFileForWrite(long size) throws IOException { String path = String.format(OUTPUT_FILE_INDEX_FORMAT_STRING, TASKTRACKER_OUTPUT); @@ -94,10 +82,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput { /** * Return a local map spill file created earlier. * - * @param spillNumber - * the number - * @return path - * @throws IOException + * @param spillNumber the number */ public Path getSpillFile(int spillNumber) throws IOException { String path = String.format(SPILL_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, spillNumber); @@ -107,12 +92,8 @@ public class LocalJobOutputFiles implements NativeTaskOutput { /** * Create a local map spill file name. * - * @param spillNumber - * the number - * @param size - * the size of the file - * @return path - * @throws IOException + * @param spillNumber the number + * @param size the size of the file */ public Path getSpillFileForWrite(int spillNumber, long size) throws IOException { String path = String.format(SPILL_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, spillNumber); @@ -122,10 +103,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput { /** * Return a local map spill index file created earlier * - * @param spillNumber - * the number - * @return path - * @throws IOException + * @param spillNumber the number */ public Path getSpillIndexFile(int spillNumber) throws IOException { String path = String @@ -136,12 +114,8 @@ public class LocalJobOutputFiles implements NativeTaskOutput { /** * Create a local map spill index file name. * - * @param spillNumber - * the number - * @param size - * the size of the file - * @return path - * @throws IOException + * @param spillNumber the number + * @param size the size of the file */ public Path getSpillIndexFileForWrite(int spillNumber, long size) throws IOException { String path = String @@ -152,10 +126,7 @@ public class LocalJobOutputFiles implements NativeTaskOutput { /** * Return a local reduce input file created earlier * - * @param mapId - * a map task id - * @return path - * @throws IOException + * @param mapId a map task id */ public Path getInputFile(int mapId) throws IOException { return lDirAlloc.getLocalPathToRead( @@ -166,14 +137,11 @@ public class LocalJobOutputFiles implements NativeTaskOutput { /** * Create a local reduce input file name. * - * @param mapId - * a map task id - * @param size - * the size of the file - * @return path - * @throws IOException + * @param mapId a map task id + * @param size the size of the file */ - public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf) throws IOException { + public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf) + throws IOException { return lDirAlloc.getLocalPathForWrite( String.format(REDUCE_INPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, mapId.getId()), size, conf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java index e92901b..7c9f433 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutput.java @@ -32,103 +32,70 @@ public interface NativeTaskOutput { /** * Return the path to local map output file created earlier - * - * @return path - * @throws IOException */ public Path getOutputFile() throws IOException; /** * Create a local map output file name. * - * @param size - * the size of the file - * @return path - * @throws IOException + * @param size the size of the file */ public Path getOutputFileForWrite(long size) throws IOException; /** * Return the path to a local map output index file created earlier - * - * @return path - * @throws IOException */ public Path getOutputIndexFile() throws IOException; /** * Create a local map output index file name. * - * @param size - * the size of the file - * @return path - * @throws IOException + * @param size the size of the file */ public Path getOutputIndexFileForWrite(long size) throws IOException; /** * Return a local map spill file created earlier. * - * @param spillNumber - * the number - * @return path - * @throws IOException + * @param spillNumber the number */ public Path getSpillFile(int spillNumber) throws IOException; /** * Create a local map spill file name. * - * @param spillNumber - * the number - * @param size - * the size of the file - * @return path - * @throws IOException + * @param spillNumber the number + * @param size the size of the file */ public Path getSpillFileForWrite(int spillNumber, long size) throws IOException; /** * Return a local map spill index file created earlier * - * @param spillNumber - * the number - * @return path - * @throws IOException + * @param spillNumber the number */ public Path getSpillIndexFile(int spillNumber) throws IOException; /** * Create a local map spill index file name. * - * @param spillNumber - * the number - * @param size - * the size of the file - * @return path - * @throws IOException + r* @param spillNumber the number + * @param size the size of the file */ public Path getSpillIndexFileForWrite(int spillNumber, long size) throws IOException; /** * Return a local reduce input file created earlier * - * @param mapId - * a map task id - * @return path - * @throws IOException + * @param mapId a map task id */ public Path getInputFile(int mapId) throws IOException; /** * Create a local reduce input file name. * - * @param mapId - * a map task id - * @param size - * the size of the file - * @return path - * @throws IOException + * @param mapId a map task id + * @param size the size of the file */ public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf) throws IOException; http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java index 5cad10b..d52ee1c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/NativeTaskOutputFiles.java @@ -28,11 +28,13 @@ import org.apache.hadoop.mapred.TaskID; /** * Manipulate the working area for the transient store for maps and reduces. - * - * This class is used by map and reduce tasks to identify the directories that they need to write - * to/read from for intermediate files. The callers of these methods are from child space and see - * mapreduce.cluster.local.dir as taskTracker/jobCache/jobId/attemptId This class should not be used - * from TaskTracker space. + * + * This class is used by map and reduce tasks to identify the directories that they need + * to write to/read from for intermediate files. The callers of these methods are from + * child space and see mapreduce.cluster.local.dir as + * taskTracker/jobCache/jobId/attemptId. + * + * This class should not be used from TaskTracker space. */ @InterfaceAudience.Private public class NativeTaskOutputFiles implements NativeTaskOutput { @@ -55,9 +57,6 @@ public class NativeTaskOutputFiles implements NativeTaskOutput { /** * Return the path to local map output file created earlier - * - * @return path - * @throws IOException */ public Path getOutputFile() throws IOException { String path = String.format(OUTPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, id); @@ -66,11 +65,8 @@ public class NativeTaskOutputFiles implements NativeTaskOutput { /** * Create a local map output file name. - * - * @param size - * the size of the file - * @return path - * @throws IOException + * + * @param size the size of the file */ public Path getOutputFileForWrite(long size) throws IOException { String path = String.format(OUTPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, id); @@ -79,9 +75,6 @@ public class NativeTaskOutputFiles implements NativeTaskOutput { /** * Return the path to a local map output index file created earlier - * - * @return path - * @throws IOException */ public Path getOutputIndexFile() throws IOException { String path = String.format(OUTPUT_FILE_INDEX_FORMAT_STRING, TASKTRACKER_OUTPUT, id); @@ -91,10 +84,7 @@ public class NativeTaskOutputFiles implements NativeTaskOutput { /** * Create a local map output index file name. * - * @param size - * the size of the file - * @return path - * @throws IOException + * @param size the size of the file */ public Path getOutputIndexFileForWrite(long size) throws IOException { String path = String.format(OUTPUT_FILE_INDEX_FORMAT_STRING, TASKTRACKER_OUTPUT, id); @@ -104,10 +94,7 @@ public class NativeTaskOutputFiles implements NativeTaskOutput { /** * Return a local map spill file created earlier. * - * @param spillNumber - * the number - * @return path - * @throws IOException + * @param spillNumber the number */ public Path getSpillFile(int spillNumber) throws IOException { String path = String.format(SPILL_FILE_FORMAT_STRING, id, TASKTRACKER_OUTPUT, spillNumber); @@ -117,12 +104,8 @@ public class NativeTaskOutputFiles implements NativeTaskOutput { /** * Create a local map spill file name. * - * @param spillNumber - * the number - * @param size - * the size of the file - * @return path - * @throws IOException + * @param spillNumber the number + * @param size the size of the file */ public Path getSpillFileForWrite(int spillNumber, long size) throws IOException { String path = String.format(SPILL_FILE_FORMAT_STRING, id, TASKTRACKER_OUTPUT, spillNumber); @@ -132,10 +115,7 @@ public class NativeTaskOutputFiles implements NativeTaskOutput { /** * Return a local map spill index file created earlier * - * @param spillNumber - * the number - * @return path - * @throws IOException + * @param spillNumber the number */ public Path getSpillIndexFile(int spillNumber) throws IOException { String path = String @@ -146,12 +126,8 @@ public class NativeTaskOutputFiles implements NativeTaskOutput { /** * Create a local map spill index file name. * - * @param spillNumber - * the number - * @param size - * the size of the file - * @return path - * @throws IOException + * @param spillNumber the number + * @param size the size of the file */ public Path getSpillIndexFileForWrite(int spillNumber, long size) throws IOException { String path = String @@ -162,10 +138,7 @@ public class NativeTaskOutputFiles implements NativeTaskOutput { /** * Return a local reduce input file created earlier * - * @param mapId - * a map task id - * @return path - * @throws IOException + * @param mapId a map task id */ public Path getInputFile(int mapId) throws IOException { return lDirAlloc.getLocalPathToRead( @@ -176,14 +149,11 @@ public class NativeTaskOutputFiles implements NativeTaskOutput { /** * Create a local reduce input file name. * - * @param mapId - * a map task id - * @param size - * the size of the file - * @return path - * @throws IOException + * @param mapId a map task id + * @param size the size of the file */ - public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf) throws IOException { + public Path getInputFileForWrite(TaskID mapId, long size, Configuration conf) + throws IOException { return lDirAlloc.getLocalPathForWrite( String.format(REDUCE_INPUT_FILE_FORMAT_STRING, TASKTRACKER_OUTPUT, mapId.getId()), size, conf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java index 017d834..03df016 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/ReadWriteBuffer.java @@ -112,10 +112,15 @@ public class ReadWriteBuffer { } public long readLong() { - final long result = ((_buff[_readPoint + 0] & 255) << 0) + ((_buff[_readPoint + 1] & 255) << 8) - + ((_buff[_readPoint + 2] & 255) << 16) + ((long) (_buff[_readPoint + 3] & 255) << 24) - + ((long) (_buff[_readPoint + 4] & 255) << 32) + ((long) (_buff[_readPoint + 5] & 255) << 40) - + ((long) (_buff[_readPoint + 6] & 255) << 48) + (((long) _buff[_readPoint + 7] << 56)); + final long result = + ((_buff[_readPoint + 0] & 255) << 0) + + ((_buff[_readPoint + 1] & 255) << 8) + + ((_buff[_readPoint + 2] & 255) << 16) + + ((long) (_buff[_readPoint + 3] & 255) << 24) + + ((long) (_buff[_readPoint + 4] & 255) << 32) + + ((long) (_buff[_readPoint + 5] & 255) << 40) + + ((long) (_buff[_readPoint + 6] & 255) << 48) + + (((long) _buff[_readPoint + 7] << 56)); _readPoint += 8; return result; @@ -144,7 +149,8 @@ public class ReadWriteBuffer { if (_buff.length - _writePoint >= toBeWritten) { return; } - final int newLength = (toBeWritten + _writePoint > CACHE_LINE_SIZE) ? (toBeWritten + _writePoint) : CACHE_LINE_SIZE; + final int newLength = (toBeWritten + _writePoint > CACHE_LINE_SIZE) ? + (toBeWritten + _writePoint) : CACHE_LINE_SIZE; final byte[] newBuff = new byte[newLength]; System.arraycopy(_buff, 0, newBuff, 0, _writePoint); _buff = newBuff; http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.cc deleted file mode 100644 index 36ff93b..0000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.cc +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright (c) 2011 Google, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// -// CityHash Version 1, by Geoff Pike and Jyrki Alakuijala -// -// This file provides CityHash64() and related functions. -// -// It's probably possible to create even faster hash functions by -// writing a program that systematically explores some of the space of -// possible hash functions, by using SIMD instructions, or by -// compromising on hash quality. - -#include "city.h" - -#include <algorithm> - -using namespace std; - -#define UNALIGNED_LOAD64(p) (*(const uint64*)(p)) -#define UNALIGNED_LOAD32(p) (*(const uint32*)(p)) - -#if !defined(LIKELY) -#if defined(__GNUC__) -#define LIKELY(x) (__builtin_expect(!!(x), 1)) -#else -#define LIKELY(x) (x) -#endif -#endif - -// Some primes between 2^63 and 2^64 for various uses. -static const uint64 k0 = 0xc3a5c85c97cb3127ULL; -static const uint64 k1 = 0xb492b66fbe98f273ULL; -static const uint64 k2 = 0x9ae16a3b2f90404fULL; -static const uint64 k3 = 0xc949d7c7509e6557ULL; - -// Bitwise right rotate. Normally this will compile to a single -// instruction, especially if the shift is a manifest constant. -static uint64 Rotate(uint64 val, int shift) { - // Avoid shifting by 64: doing so yields an undefined result. - return shift == 0 ? val : ((val >> shift) | (val << (64 - shift))); -} - -// Equivalent to Rotate(), but requires the second arg to be non-zero. -// On x86-64, and probably others, it's possible for this to compile -// to a single instruction if both args are already in registers. -static uint64 RotateByAtLeast1(uint64 val, int shift) { - return (val >> shift) | (val << (64 - shift)); -} - -static uint64 ShiftMix(uint64 val) { - return val ^ (val >> 47); -} - -static uint64 HashLen16(uint64 u, uint64 v) { - return Hash128to64(uint128(u, v)); -} - -static uint64 HashLen0to16(const char *s, size_t len) { - if (len > 8) { - uint64 a = UNALIGNED_LOAD64(s); - uint64 b = UNALIGNED_LOAD64(s + len - 8); - return HashLen16(a, RotateByAtLeast1(b + len, len)) ^ b; - } - if (len >= 4) { - uint64 a = UNALIGNED_LOAD32(s); - return HashLen16(len + (a << 3), UNALIGNED_LOAD32(s + len - 4)); - } - if (len > 0) { - uint8 a = s[0]; - uint8 b = s[len >> 1]; - uint8 c = s[len - 1]; - uint32 y = static_cast<uint32>(a) + (static_cast<uint32>(b) << 8); - uint32 z = len + (static_cast<uint32>(c) << 2); - return ShiftMix(y * k2 ^ z * k3) * k2; - } - return k2; -} - -// This probably works well for 16-byte strings as well, but it may be overkill -// in that case. -static uint64 HashLen17to32(const char *s, size_t len) { - uint64 a = UNALIGNED_LOAD64(s) * k1; - uint64 b = UNALIGNED_LOAD64(s + 8); - uint64 c = UNALIGNED_LOAD64(s + len - 8) * k2; - uint64 d = UNALIGNED_LOAD64(s + len - 16) * k0; - return HashLen16(Rotate(a - b, 43) + Rotate(c, 30) + d, - a + Rotate(b ^ k3, 20) - c + len); -} - -// Return a 16-byte hash for 48 bytes. Quick and dirty. -// Callers do best to use "random-looking" values for a and b. -static pair<uint64, uint64> WeakHashLen32WithSeeds( - uint64 w, uint64 x, uint64 y, uint64 z, uint64 a, uint64 b) { - a += w; - b = Rotate(b + a + z, 21); - uint64 c = a; - a += x; - a += y; - b += Rotate(a, 44); - return make_pair(a + z, b + c); -} - -// Return a 16-byte hash for s[0] ... s[31], a, and b. Quick and dirty. -static pair<uint64, uint64> WeakHashLen32WithSeeds( - const char* s, uint64 a, uint64 b) { - return WeakHashLen32WithSeeds(UNALIGNED_LOAD64(s), - UNALIGNED_LOAD64(s + 8), - UNALIGNED_LOAD64(s + 16), - UNALIGNED_LOAD64(s + 24), - a, - b); -} - -// Return an 8-byte hash for 33 to 64 bytes. -static uint64 HashLen33to64(const char *s, size_t len) { - uint64 z = UNALIGNED_LOAD64(s + 24); - uint64 a = UNALIGNED_LOAD64(s) + (len + UNALIGNED_LOAD64(s + len - 16)) * k0; - uint64 b = Rotate(a + z, 52); - uint64 c = Rotate(a, 37); - a += UNALIGNED_LOAD64(s + 8); - c += Rotate(a, 7); - a += UNALIGNED_LOAD64(s + 16); - uint64 vf = a + z; - uint64 vs = b + Rotate(a, 31) + c; - a = UNALIGNED_LOAD64(s + 16) + UNALIGNED_LOAD64(s + len - 32); - z = UNALIGNED_LOAD64(s + len - 8); - b = Rotate(a + z, 52); - c = Rotate(a, 37); - a += UNALIGNED_LOAD64(s + len - 24); - c += Rotate(a, 7); - a += UNALIGNED_LOAD64(s + len - 16); - uint64 wf = a + z; - uint64 ws = b + Rotate(a, 31) + c; - uint64 r = ShiftMix((vf + ws) * k2 + (wf + vs) * k0); - return ShiftMix(r * k0 + vs) * k2; -} - -uint64 CityHash64(const char *s, size_t len) { - if (len <= 32) { - if (len <= 16) { - return HashLen0to16(s, len); - } else { - return HashLen17to32(s, len); - } - } else if (len <= 64) { - return HashLen33to64(s, len); - } - - // For strings over 64 bytes we hash the end first, and then as we - // loop we keep 56 bytes of state: v, w, x, y, and z. - uint64 x = UNALIGNED_LOAD64(s); - uint64 y = UNALIGNED_LOAD64(s + len - 16) ^ k1; - uint64 z = UNALIGNED_LOAD64(s + len - 56) ^ k0; - pair<uint64, uint64> v = WeakHashLen32WithSeeds(s + len - 64, len, y); - pair<uint64, uint64> w = WeakHashLen32WithSeeds(s + len - 32, len * k1, k0); - z += ShiftMix(v.second) * k1; - x = Rotate(z + x, 39) * k1; - y = Rotate(y, 33) * k1; - - // Decrease len to the nearest multiple of 64, and operate on 64-byte chunks. - len = (len - 1) & ~static_cast<size_t>(63); - do { - x = Rotate(x + y + v.first + UNALIGNED_LOAD64(s + 16), 37) * k1; - y = Rotate(y + v.second + UNALIGNED_LOAD64(s + 48), 42) * k1; - x ^= w.second; - y ^= v.first; - z = Rotate(z ^ w.first, 33); - v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first); - w = WeakHashLen32WithSeeds(s + 32, z + w.second, y); - std::swap(z, x); - s += 64; - len -= 64; - } while (len != 0); - return HashLen16(HashLen16(v.first, w.first) + ShiftMix(y) * k1 + z, - HashLen16(v.second, w.second) + x); -} - -uint64 CityHash64WithSeed(const char *s, size_t len, uint64 seed) { - return CityHash64WithSeeds(s, len, k2, seed); -} - -uint64 CityHash64WithSeeds(const char *s, size_t len, - uint64 seed0, uint64 seed1) { - return HashLen16(CityHash64(s, len) - seed0, seed1); -} - -// A subroutine for CityHash128(). Returns a decent 128-bit hash for strings -// of any length representable in ssize_t. Based on City and Murmur. -static uint128 CityMurmur(const char *s, size_t len, uint128 seed) { - uint64 a = Uint128Low64(seed); - uint64 b = Uint128High64(seed); - uint64 c = 0; - uint64 d = 0; - ssize_t l = len - 16; - if (l <= 0) { // len <= 16 - c = b * k1 + HashLen0to16(s, len); - d = Rotate(a + (len >= 8 ? UNALIGNED_LOAD64(s) : c), 32); - } else { // len > 16 - c = HashLen16(UNALIGNED_LOAD64(s + len - 8) + k1, a); - d = HashLen16(b + len, c + UNALIGNED_LOAD64(s + len - 16)); - a += d; - do { - a ^= ShiftMix(UNALIGNED_LOAD64(s) * k1) * k1; - a *= k1; - b ^= a; - c ^= ShiftMix(UNALIGNED_LOAD64(s + 8) * k1) * k1; - c *= k1; - d ^= c; - s += 16; - l -= 16; - } while (l > 0); - } - a = HashLen16(a, c); - b = HashLen16(d, b); - return uint128(a ^ b, HashLen16(b, a)); -} - -uint128 CityHash128WithSeed(const char *s, size_t len, uint128 seed) { - if (len < 128) { - return CityMurmur(s, len, seed); - } - - // We expect len >= 128 to be the common case. Keep 56 bytes of state: - // v, w, x, y, and z. - pair<uint64, uint64> v, w; - uint64 x = Uint128Low64(seed); - uint64 y = Uint128High64(seed); - uint64 z = len * k1; - v.first = Rotate(y ^ k1, 49) * k1 + UNALIGNED_LOAD64(s); - v.second = Rotate(v.first, 42) * k1 + UNALIGNED_LOAD64(s + 8); - w.first = Rotate(y + z, 35) * k1 + x; - w.second = Rotate(x + UNALIGNED_LOAD64(s + 88), 53) * k1; - - // This is the same inner loop as CityHash64(), manually unrolled. - do { - x = Rotate(x + y + v.first + UNALIGNED_LOAD64(s + 16), 37) * k1; - y = Rotate(y + v.second + UNALIGNED_LOAD64(s + 48), 42) * k1; - x ^= w.second; - y ^= v.first; - z = Rotate(z ^ w.first, 33); - v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first); - w = WeakHashLen32WithSeeds(s + 32, z + w.second, y); - std::swap(z, x); - s += 64; - x = Rotate(x + y + v.first + UNALIGNED_LOAD64(s + 16), 37) * k1; - y = Rotate(y + v.second + UNALIGNED_LOAD64(s + 48), 42) * k1; - x ^= w.second; - y ^= v.first; - z = Rotate(z ^ w.first, 33); - v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first); - w = WeakHashLen32WithSeeds(s + 32, z + w.second, y); - std::swap(z, x); - s += 64; - len -= 128; - } while (LIKELY(len >= 128)); - y += Rotate(w.first, 37) * k0 + z; - x += Rotate(v.first + z, 49) * k0; - // If 0 < len < 128, hash up to 4 chunks of 32 bytes each from the end of s. - for (size_t tail_done = 0; tail_done < len; ) { - tail_done += 32; - y = Rotate(y - x, 42) * k0 + v.second; - w.first += UNALIGNED_LOAD64(s + len - tail_done + 16); - x = Rotate(x, 49) * k0 + w.first; - w.first += v.first; - v = WeakHashLen32WithSeeds(s + len - tail_done, v.first, v.second); - } - // At this point our 48 bytes of state should contain more than - // enough information for a strong 128-bit hash. We use two - // different 48-byte-to-8-byte hashes to get a 16-byte final result. - x = HashLen16(x, v.first); - y = HashLen16(y, w.first); - return uint128(HashLen16(x + v.second, w.second) + y, - HashLen16(x + w.second, y + v.second)); -} - -uint128 CityHash128(const char *s, size_t len) { - if (len >= 16) { - return CityHash128WithSeed(s + 16, - len - 16, - uint128(UNALIGNED_LOAD64(s) ^ k3, - UNALIGNED_LOAD64(s + 8))); - } else if (len >= 8) { - return CityHash128WithSeed(NULL, - 0, - uint128(UNALIGNED_LOAD64(s) ^ (len * k0), - UNALIGNED_LOAD64(s + len - 8) ^ k1)); - } else { - return CityHash128WithSeed(s, len, uint128(k0, k1)); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.h deleted file mode 100644 index 7b290be..0000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/cityhash/city.h +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2011 Google, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// -// CityHash Version 1, by Geoff Pike and Jyrki Alakuijala -// -// This file provides a few functions for hashing strings. On x86-64 -// hardware in 2011, CityHash64() is faster than other high-quality -// hash functions, such as Murmur. This is largely due to higher -// instruction-level parallelism. CityHash64() and CityHash128() also perform -// well on hash-quality tests. -// -// CityHash128() is optimized for relatively long strings and returns -// a 128-bit hash. For strings more than about 2000 bytes it can be -// faster than CityHash64(). -// -// Functions in the CityHash family are not suitable for cryptography. -// -// WARNING: This code has not been tested on big-endian platforms! -// It is known to work well on little-endian platforms that have a small penalty -// for unaligned reads, such as current Intel and AMD moderate-to-high-end CPUs. -// -// By the way, for some hash functions, given strings a and b, the hash -// of a+b is easily derived from the hashes of a and b. This property -// doesn't hold for any hash functions in this file. - -#ifndef CITY_HASH_H_ -#define CITY_HASH_H_ - -#include <stdlib.h> // for size_t. -#include <stdint.h> -#include <utility> - -typedef uint8_t uint8; -typedef uint32_t uint32; -typedef uint64_t uint64; -typedef std::pair<uint64, uint64> uint128; - -inline uint64 Uint128Low64(const uint128& x) { return x.first; } -inline uint64 Uint128High64(const uint128& x) { return x.second; } - -// Hash function for a byte array. -uint64 CityHash64(const char *buf, size_t len); - -// Hash function for a byte array. For convenience, a 64-bit seed is also -// hashed into the result. -uint64 CityHash64WithSeed(const char *buf, size_t len, uint64 seed); - -// Hash function for a byte array. For convenience, two seeds are also -// hashed into the result. -uint64 CityHash64WithSeeds(const char *buf, size_t len, - uint64 seed0, uint64 seed1); - -// Hash function for a byte array. -uint128 CityHash128(const char *s, size_t len); - -// Hash function for a byte array. For convenience, a 128-bit seed is also -// hashed into the result. -uint128 CityHash128WithSeed(const char *s, size_t len, uint128 seed); - -// Hash 128 input bits down to 64 bits of output. -// This is intended to be a reasonably good hash function. -inline uint64 Hash128to64(const uint128& x) { - // Murmur-inspired hashing. - const uint64 kMul = 0x9ddfea08eb382d69ULL; - uint64 a = (Uint128Low64(x) ^ Uint128High64(x)) * kMul; - a ^= (a >> 47); - uint64 b = (Uint128High64(x) ^ a) * kMul; - b ^= (b >> 47); - b *= kMul; - return b; -} - -#endif // CITY_HASH_H_
