http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.cc deleted file mode 100644 index eec8091..0000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.cc +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "Hash.h" - -namespace NativeTask { - -} // namespace NativeTask -
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.h deleted file mode 100644 index 4f99d26..0000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.h +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef HASH_H_ -#define HASH_H_ - -#include <stdint.h> -#include <stdlib.h> - -// Hash function for a byte array. -extern uint64_t CityHash64(const char *buf, size_t len); - -// Hash function for a byte array. For convenience, a 64-bit seed is also -// hashed into the result. -extern uint64_t CityHash64WithSeed(const char *buf, size_t len, uint64_t seed); - -namespace NativeTask { - -class Hash { -public: - /** - * Compatible with hadoop Text & BytesWritable hash - */ - inline static int32_t BytesHash(const char * bytes, uint32_t length) { - int32_t hash = 1; - for (uint32_t i = 0; i < length; i++) - hash = (31 * hash) + (int32_t)bytes[i]; - return hash; - } - - /** - * Unsigned version of BytesHash - */ - inline static uint32_t BytesHashU(const char * bytes, uint32_t length) { - uint32_t hash = 1; - for (uint32_t i = 0; i < length; i++) - hash = (31U * hash) + (uint32_t)bytes[i]; - return hash; - } - - /** - * City hash, faster for longer input - */ - inline static uint64_t CityHash(const char * bytes, uint32_t length) { - return CityHash64(bytes, length); - } - - /** - * City hash, faster for longer input - */ - inline static uint64_t CityHashWithSeed(const char * bytes, uint32_t length, uint64_t seed) { - return CityHash64WithSeed(bytes, length, seed); - } -}; - -} // namespace NativeTask - -#endif /* HASH_H_ */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Random.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Random.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Random.cc index e6cfec2..6266b1f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Random.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Random.cc @@ -17,8 +17,8 @@ */ #include <math.h> -#include "commons.h" -#include "Random.h" +#include "lib/commons.h" +#include "util/Random.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc index 7baaa0c..3ed7435 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/StringUtil.cc @@ -17,8 +17,8 @@ */ #include <stdarg.h> -#include "commons.h" -#include "StringUtil.h" +#include "lib/commons.h" +#include "util/StringUtil.h" namespace NativeTask { @@ -201,8 +201,9 @@ bool StringUtil::StartsWith(const string & str, const string & prefix) { } bool StringUtil::EndsWith(const string & str, const string & suffix) { - if ((suffix.length() > str.length()) - || (memcmp(str.data() + str.length() - suffix.length(), suffix.data(), suffix.length()) != 0)) { + if ((suffix.length() > str.length()) || + (memcmp(str.data() + str.length() - suffix.length(), + suffix.data(), suffix.length()) != 0)) { return false; } return true; http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.cc index c3fb307..357a68d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.cc @@ -16,10 +16,10 @@ * limitations under the License. */ -#include "commons.h" -#include "jniutils.h" -#include "StringUtil.h" -#include "SyncUtils.h" +#include "lib/commons.h" +#include "lib/jniutils.h" +#include "util/StringUtil.h" +#include "util/SyncUtils.h" namespace NativeTask { @@ -52,110 +52,4 @@ void Lock::unlock() { PthreadCall("unlock", pthread_mutex_unlock(&_mutex)); } -#ifdef __MACH__ -SpinLock::SpinLock() : _spin(0) { -} - -SpinLock::~SpinLock() { - -} - -void SpinLock::lock() { - OSSpinLockLock(&_spin); -} - -void SpinLock::unlock() { - OSSpinLockUnlock(&_spin); -} -#else -SpinLock::SpinLock() { - PthreadCall("init mutex", pthread_spin_init(&_spin, 0)); -} - -SpinLock::~SpinLock() { - PthreadCall("destroy mutex", pthread_spin_destroy(&_spin)); -} - -void SpinLock::lock() { - PthreadCall("lock", pthread_spin_lock(&_spin)); -} - -void SpinLock::unlock() { - PthreadCall("unlock", pthread_spin_unlock(&_spin)); -} -#endif - -Condition::Condition(Lock* mu) - : _lock(mu) { - PthreadCall("init cv", pthread_cond_init(&_condition, NULL)); -} - -Condition::~Condition() { - PthreadCall("destroy cv", pthread_cond_destroy(&_condition)); -} - -void Condition::wait() { - PthreadCall("wait", pthread_cond_wait(&_condition, &_lock->_mutex)); -} - -void Condition::signal() { - PthreadCall("signal", pthread_cond_signal(&_condition)); -} - -void Condition::signalAll() { - PthreadCall("broadcast", pthread_cond_broadcast(&_condition)); -} - -void * Thread::ThreadRunner(void * pthis) { - try { - ((Thread*)pthis)->run(); - } catch (std::exception & e) { - LOG("err!!!! %s", e.what()); - } - return NULL; -} - -Thread::Thread() - : _thread((pthread_t)0), // safe for linux & macos - _runable(NULL) { -} - -Thread::Thread(Runnable * runnable) - : _thread((pthread_t)0), _runable(runnable) { -} - -void Thread::setTask(const Runnable & runnable) { - _runable = const_cast<Runnable*>(&runnable); -} - -Thread::~Thread() { - -} - -void Thread::start() { - PthreadCall("pthread_create", pthread_create(&_thread, NULL, ThreadRunner, this)); -} - -void Thread::join() { - PthreadCall("pthread_join", pthread_join(_thread, NULL)); -} - -void Thread::stop() { - PthreadCall("pthread_cancel", pthread_cancel(_thread)); -} - -void Thread::run() { - if (_runable != NULL) { - _runable->run(); - } -} - -void Thread::EnableJNI() { - JNU_AttachCurrentThread(); -} - -void Thread::ReleaseJNI() { - JNU_DetachCurrentThread(); -} - } // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.h index 15ce2c8..270f851 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/SyncUtils.h @@ -47,38 +47,6 @@ private: void operator=(const Lock&); }; -class SpinLock { -public: - SpinLock(); - ~SpinLock(); - - void lock(); - void unlock(); - -private: -#ifdef __MACH__ - OSSpinLock _spin; -#else - pthread_spinlock_t _spin; -#endif - - // No copying - SpinLock(const Lock&); - void operator=(const Lock&); -}; - -class Condition { -public: - explicit Condition(Lock* mu); - ~Condition(); - void wait(); - void signal(); - void signalAll(); -private: - pthread_cond_t _condition; - Lock* _lock; -}; - template<typename LockT> class ScopeLock { public: @@ -97,202 +65,6 @@ private: void operator=(const ScopeLock&); }; -class Runnable { -public: - virtual ~Runnable() { - } - virtual void run() = 0; -}; - -class Thread : public Runnable { -protected: - pthread_t _thread; - Runnable * _runable; -public: - Thread(); - Thread(Runnable * runnable); - virtual ~Thread(); - - void setTask(const Runnable & runnable); - void start(); - void join(); - void stop(); - virtual void run(); - - /** - * Enable JNI for current thread - */ - static void EnableJNI(); - /** - * Release JNI when thread is at end if current - * thread called EnableJNI before - */ - static void ReleaseJNI(); -private: - static void * ThreadRunner(void * pthis); -}; - -// Sure <tr1/functional> is better -template<typename Subject, typename Method> -class FunctionRunner : public Runnable { -protected: - Subject & _subject; - Method _method; -public: - FunctionRunner(Subject & subject, Method method) - : _subject(subject), _method(method) { - } - - virtual void run() { - (_subject.*_method)(); - } -}; - -template<typename Subject, typename Method, typename Arg> -class FunctionRunner1 : public Runnable { -protected: - Subject & _subject; - Method _method; - Arg _arg; -public: - FunctionRunner1(Subject & subject, Method method, Arg arg) - : _subject(subject), _method(method), _arg(arg) { - } - - virtual void run() { - (_subject.*_method)(_arg); - } -}; - -template<typename Subject, typename Method, typename Arg1, typename Arg2> -class FunctionRunner2 : public Runnable { -protected: - Subject & _subject; - Method _method; - Arg1 _arg1; - Arg2 _arg2; -public: - FunctionRunner2(Subject & subject, Method method, Arg1 arg1, Arg2 arg2) - : _subject(subject), _method(method), _arg1(arg1), _arg2(arg2) { - } - - virtual void run() { - (_subject.*_method)(_arg1, _arg2); - } -}; - -template<typename Subject, typename Method> -inline FunctionRunner<Subject, Method> * BindNew(Subject & subject, Method method) { - return new FunctionRunner<Subject, Method>(subject, method); -} - -template<typename Subject, typename Method, typename Arg> -inline FunctionRunner1<Subject, Method, Arg> * BindNew(Subject & subject, Method method, Arg arg) { - return new FunctionRunner1<Subject, Method, Arg>(subject, method, arg); -} - -template<typename Subject, typename Method, typename Arg1, typename Arg2> -inline FunctionRunner2<Subject, Method, Arg1, Arg2> * BindNew(Subject & subject, Method method, - Arg1 arg1, Arg2 arg2) { - return new FunctionRunner2<Subject, Method, Arg1, Arg2>(subject, method, arg1, arg2); -} - -class ConcurrentIndex { -private: - size_t _index; - size_t _end; - SpinLock _lock; -public: - ConcurrentIndex(size_t count) - : _index(0), _end(count) { - } - - ConcurrentIndex(size_t start, size_t end) - : _index(start), _end(end) { - } - - size_t count() { - return _end; - } - - ssize_t next() { - ScopeLock<SpinLock> autoLock(_lock); - if (_index == _end) { - return -1; - } else { - ssize_t ret = _index; - _index++; - return ret; - } - } -}; - -template<typename Subject, typename Method, typename RangeType> -class ParallelForWorker : public Runnable { -protected: - ConcurrentIndex * _index; - Subject * _subject; - Method _method; -public: - ParallelForWorker() - : _index(NULL), _subject(NULL) { - } - - ParallelForWorker(ConcurrentIndex * index, Subject * subject, Method method) - : _index(index), _subject(subject), _method(method) { - } - - void reset(ConcurrentIndex * index, Subject * subject, Method method) { - _index = index; - _subject = subject; - _method = method; - } - - virtual void run() { - ssize_t i; - while ((i = _index->next()) >= 0) { - (_subject->*_method)(i); - } - } -}; - -template<typename Subject, typename Method, typename RangeType> -void ParallelFor(Subject & subject, Method method, RangeType begin, RangeType end, - size_t thread_num) { - RangeType count = end - begin; - if (thread_num <= 1 || count <= 1) { - for (RangeType i = begin; i < end; i++) { - (subject.*method)(i); - } - } else if (thread_num == 2) { - ConcurrentIndex index = ConcurrentIndex(begin, end); - ParallelForWorker<Subject, Method, RangeType> workers[2]; - Thread sideThread; - workers[0].reset(&index, &subject, method); - workers[1].reset(&index, &subject, method); - sideThread.setTask(workers[0]); - sideThread.start(); - workers[1].run(); - sideThread.join(); - } else { - ConcurrentIndex index = ConcurrentIndex(begin, end); - ParallelForWorker<Subject, Method, RangeType> * workers = new ParallelForWorker<Subject, Method, - RangeType> [thread_num]; - Thread * threads = new Thread[thread_num - 1]; - for (size_t i = 0; i < thread_num - 1; i++) { - workers[i].reset(&index, &subject, method); - threads[i].setTask(workers[i]); - threads[i].start(); - } - workers[thread_num - 1].reset(&index, &subject, method); - workers[thread_num - 1].run(); - for (size_t i = 0; i < thread_num - 1; i++) { - threads[i].join(); - } - delete[] threads; - delete[] workers; - } -} } // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Timer.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Timer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Timer.cc index b1d3a00..9f8a9ad 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Timer.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Timer.cc @@ -17,9 +17,9 @@ */ #include <time.h> -#include "commons.h" -#include "StringUtil.h" -#include "Timer.h" +#include "lib/commons.h" +#include "util/StringUtil.h" +#include "util/Timer.h" namespace NativeTask { @@ -51,7 +51,6 @@ Timer::Timer() { } Timer::~Timer() { - } uint64_t Timer::last() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc index a16c654..2436681 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.cc @@ -16,9 +16,9 @@ * limitations under the License. */ -#include "commons.h" -#include "StringUtil.h" -#include "WritableUtils.h" +#include "lib/commons.h" +#include "util/StringUtil.h" +#include "util/WritableUtils.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.h index 3e93cbe..24a2b1b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/WritableUtils.h @@ -21,7 +21,7 @@ #include <stdint.h> #include <string> -#include "Streams.h" +#include "lib/Streams.h" #include "NativeTask.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc index bcc3b3c..c09b7b0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCommand.cc @@ -16,13 +16,13 @@ * limitations under the License. */ -#include "commons.h" -#include "BufferStream.h" -#include "Buffers.h" +#include "lib/commons.h" +#include "lib/BufferStream.h" +#include "lib/Buffers.h" #include "test_commons.h" #include "NativeTask.h" -using namespace NativeTask; +namespace NativeTask { TEST(Command, equals) { Command cmd1(100, "hello command"); @@ -36,3 +36,4 @@ TEST(Command, equals) { ASSERT_EQ(helloCommand, cmd1.description()); } +} // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc index 1b99286..387e262 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCompressions.cc @@ -18,11 +18,11 @@ #include "lz4.h" #include "config.h" -#include "commons.h" -#include "Path.h" -#include "BufferStream.h" -#include "FileSystem.h" -#include "Compressions.h" +#include "lib/commons.h" +#include "lib/Path.h" +#include "lib/BufferStream.h" +#include "lib/FileSystem.h" +#include "lib/Compressions.h" #include "test_commons.h" #if defined HADOOP_SNAPPY_LIBRARY @@ -144,7 +144,7 @@ TEST(Perf, CompressionUtil) { } class CompressResult { -public: + public: uint64_t uncompressedSize; uint64_t compressedSize; uint64_t compressTime; @@ -193,10 +193,8 @@ void MeasureSingleFileLz4(const string & path, CompressResult & total, size_t bl result.compressTime += endTime - startTime; startTime = t.now(); for (int i = 0; i < times; i++) { -// memset(dest, 0, currentblocksize+8); int osize = LZ4_uncompress(outputBuffer, dest, currentblocksize); ASSERT_EQ(currentblocksize, osize); -// printf("%016llx blocksize: %lu\n", bswap64(*(uint64_t*)(dest+currentblocksize)), currentblocksize); } endTime = t.now(); result.uncompressTime += endTime - startTime; http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestConfig.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestConfig.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestConfig.cc index 36315ce..a5c0b9a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestConfig.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestConfig.cc @@ -16,9 +16,9 @@ * limitations under the License. */ -#include "commons.h" -#include "BufferStream.h" -#include "Buffers.h" +#include "lib/commons.h" +#include "lib/BufferStream.h" +#include "lib/Buffers.h" #include "test_commons.h" float absoute(float v) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc index d4550bc..028c43b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestCounter.cc @@ -16,10 +16,10 @@ * limitations under the License. */ -#include "commons.h" -#include "NativeObjectFactory.h" -#include "BufferStream.h" -#include "Buffers.h" +#include "lib/commons.h" +#include "lib/NativeObjectFactory.h" +#include "lib/BufferStream.h" +#include "lib/Buffers.h" #include "test_commons.h" TEST(Counter, Counter) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc index 8b5a76c..667858b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestFileSystem.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "FileSystem.h" +#include "lib/FileSystem.h" #include "test_commons.h" TEST(FileSystem, RawFileSystem) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc index 955ef29..e1e32d4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestIFile.cc @@ -17,11 +17,11 @@ */ #include <algorithm> -#include "commons.h" +#include "lib/commons.h" #include "config.h" -#include "BufferStream.h" -#include "FileSystem.h" -#include "IFile.h" +#include "lib/BufferStream.h" +#include "lib/FileSystem.h" +#include "lib/IFile.h" #include "test_commons.h" SingleSpillInfo * writeIFile(int partition, vector<pair<string, string> > & kvs, @@ -166,8 +166,8 @@ TEST(Perf, IFile) { delete[] buff; } -// The Glibc has a bug in the file tell api, it will overwrite the file data -// unexpected. +// The Glibc has a bug in the file tell api, it will overwrite the file data +// unexpected. // Please check https://rhn.redhat.com/errata/RHBA-2013-0279.html // This case is to check whether the bug exists. // If it exists, it means you need to upgrade the glibc. @@ -189,12 +189,11 @@ TEST(IFile, TestGlibCBug) { uint32_t length = 0; reader->nextPartition(); uint32_t index = 0; - while(NULL != (key = reader->nextKey(length))) { + while (NULL != (key = reader->nextKey(length))) { int32_t realKey = (int32_t)bswap(*(uint32_t *)(key)); ASSERT_LT(index, 5); ASSERT_EQ(expect[index], realKey); index++; } delete reader; - } http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc index c62232a..8af4c8a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestMain.cc @@ -23,10 +23,10 @@ #endif #include <stdexcept> -#include "commons.h" -#include "Buffers.h" -#include "FileSystem.h" -#include "NativeObjectFactory.h" +#include "lib/commons.h" +#include "lib/Buffers.h" +#include "lib/FileSystem.h" +#include "lib/NativeObjectFactory.h" #include "test_commons.h" extern "C" { @@ -52,11 +52,11 @@ void handler(int sig) { } } -using namespace NativeTask; - typedef char * CString; -int main(int argc, char ** argv) { +namespace NativeTask { + +int DoMain(int argc, char** argv) { signal(SIGSEGV, handler); CString * newArgv = new CString[argc + 1]; memcpy(newArgv, argv, argc * sizeof(CString)); @@ -107,3 +107,10 @@ int main(int argc, char ** argv) { return 1; } } + +} // namespace NativeTask + + +int main(int argc, char ** argv) { + return NativeTask::DoMain(argc, argv); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestPrimitives.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestPrimitives.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestPrimitives.cc index 5a549c5..b2051c7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestPrimitives.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestPrimitives.cc @@ -166,7 +166,7 @@ inline char * memchrbrf2(char * p, char ch, size_t len) { // not safe in MACOSX, segment fault, should be safe on Linux with out mmap inline int memchr_sse(const char *s, int c, int len) { - //len : edx; c: esi; s:rdi + // len : edx; c: esi; s:rdi int index = 0; #ifdef __X64 http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc index 2403db9..1c391a6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/TestSort.cc @@ -16,10 +16,10 @@ * limitations under the License. */ -#include "commons.h" -#include "Streams.h" -#include "Buffers.h" -#include "DualPivotQuickSort.h" +#include "lib/commons.h" +#include "lib/Streams.h" +#include "lib/Buffers.h" +#include "util/DualPivotQuickSort.h" #include "test_commons.h" string gBuffer; @@ -82,7 +82,7 @@ static int compare_offset(const void * plh, const void * prh) { * dualpivot sort compare function */ class CompareOffset { -public: + public: int64_t operator()(uint32_t lhs, uint32_t rhs) { KVBuffer * lhb = (KVBuffer*)get_position(lhs); @@ -101,7 +101,7 @@ public: * quicksort compare function */ class OffsetLessThan { -public: + public: bool operator()(uint32_t lhs, uint32_t rhs) { KVBuffer * lhb = (KVBuffer*)get_position(lhs); KVBuffer * rhb = (KVBuffer*)get_position(rhs); @@ -132,7 +132,7 @@ static int compare_offset2(const void * plh, const void * prh) { * dualpivot sort compare function */ class CompareOffset2 { -public: + public: int64_t operator()(uint32_t lhs, uint32_t rhs) { KVBuffer * lhb = (KVBuffer*)get_position(lhs); @@ -151,7 +151,7 @@ public: * quicksort compare function */ class OffsetLessThan2 { -public: + public: bool operator()(uint32_t lhs, uint32_t rhs) { KVBuffer * lhb = (KVBuffer*)get_position(lhs); @@ -163,23 +163,6 @@ public: } }; -/* -void makeInput(string & dest, vector<uint32_t> & offsets, uint64_t length) { - TeraGen tera = TeraGen(length / 100, 1, 0); - dest.reserve(length + 1024); - string k, v; - while (tera.next(k, v)) { - offsets.push_back(dest.length()); - uint32_t tempLen = k.length(); - dest.append((const char *)&tempLen, 4); - dest.append(k.data(), k.length()); - tempLen = v.length(); - dest.append((const char *)&tempLen, 4); - dest.append(v.data(), v.length()); - } -} -*/ - void makeInputWord(string & dest, vector<uint32_t> & offsets, uint64_t length) { Random r; dest.reserve(length + 1024); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteArray.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteArray.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteArray.cc index f20d67f..1b26737 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteArray.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteArray.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "lib/Combiner.h" #include "test_commons.h" #include <iostream> http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc index c03f02e..02acdda 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestByteBuffer.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "lib/Combiner.h" #include "test_commons.h" #include <iostream> http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc index 9572d87..f9e980e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForDualPivotQuickSort.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "lib/Combiner.h" #include "lib/MemoryBlock.h" #include "test_commons.h" http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc index 6dd97ba..1a914b6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestComparatorForStdSort.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "lib/Combiner.h" #include "lib/MemoryBlock.h" #include "test_commons.h" http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc index a2dd909..d1b5c37 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestFixSizeContainer.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "lib/Combiner.h" #include "test_commons.h" #include <iostream> http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc index 88a3258..d25cd60 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestIterator.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "lib/Combiner.h" #include "test_commons.h" #include <iostream> @@ -30,7 +30,7 @@ class MockIterator : public KVIterator { std::map<int, int> expectkeyCountMap; char buffer[8]; -public: + public: MockIterator() : index(0) { kvs.push_back(std::pair<int, int>(10, 100)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc index 32b5891..e47e169 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestKVBuffer.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "lib/Combiner.h" #include "test_commons.h" #include <iostream> http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc index c11d5ec..8d784fb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemBlockIterator.cc @@ -16,13 +16,11 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "test_commons.h" -#include "MapOutputSpec.h" +#include "lib/MapOutputSpec.h" #include "lib/MemoryBlock.h" -using namespace NativeTask; - namespace NativeTask { TEST(MemoryBlockIterator, test) { @@ -46,7 +44,7 @@ TEST(MemoryBlockIterator, test) { } class MemoryBlockFactory { -public: + public: static MemoryBlock * create(std::vector<int> & keys) { const uint32_t BUFFER_LENGTH = 1000; char * bytes = new char[BUFFER_LENGTH]; @@ -65,7 +63,6 @@ public: } return block1; } - }; TEST(MemoryBlockIterator, compare) { @@ -109,5 +106,5 @@ TEST(MemoryBlockIterator, compare) { delete block2; delete block1; } -} +} // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc index 1d48c79..6af73c5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryBlock.cc @@ -16,12 +16,11 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "test_commons.h" -#include "MapOutputSpec.h" +#include "lib/MapOutputSpec.h" #include "lib/MemoryBlock.h" -using namespace NativeTask; namespace NativeTaskTest { TEST(MemoryBlock, test) { @@ -107,5 +106,4 @@ TEST(MemoryBlock, sort) { delete [] bytes; } -} - +} // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryPool.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryPool.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryPool.cc index 3c67332..e33421d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryPool.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestMemoryPool.cc @@ -16,14 +16,12 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "test_commons.h" -#include "PartitionBucket.h" -#include "PartitionBucketIterator.h" -#include "MemoryBlock.h" -#include "IFile.h" - -using namespace NativeTask; +#include "lib/PartitionBucket.h" +#include "lib/PartitionBucketIterator.h" +#include "lib/MemoryBlock.h" +#include "lib/IFile.h" namespace NativeTask { @@ -47,4 +45,4 @@ TEST(MemoryPool, general) { delete pool; } -} +} // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc index a975243..79e1b5e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestPartitionBucket.cc @@ -16,24 +16,22 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "test_commons.h" -#include "PartitionBucket.h" -#include "PartitionBucketIterator.h" -#include "MemoryBlock.h" -#include "IFile.h" - -using namespace NativeTask; +#include "lib/PartitionBucket.h" +#include "lib/PartitionBucketIterator.h" +#include "lib/MemoryBlock.h" +#include "lib/IFile.h" namespace NativeTask { class MockIFileWriter : public IFileWriter { -private: + private: char * _buff; uint32_t _position; uint32_t _capacity; -public: + public: MockIFileWriter(char * buff, uint32_t capacity) : IFileWriter(NULL, CHECKSUM_NONE, TextType, TextType, "", NULL), _buff(buff), _position(0), _capacity(capacity) { @@ -55,8 +53,8 @@ public: TEST(PartitionBucket, general) { MemoryPool * pool = new MemoryPool(); - const uint32_t POOL_SIZE = 1024 * 1024; //1MB - const uint32_t BLOCK_SIZE = 1024; //1KB + const uint32_t POOL_SIZE = 1024 * 1024; // 1MB + const uint32_t BLOCK_SIZE = 1024; // 1KB const uint32_t PARTITION_ID = 3; pool->init(POOL_SIZE); ComparatorPtr comparator = NativeTask::get_comparator(BytesType, NULL); @@ -74,8 +72,8 @@ TEST(PartitionBucket, general) { TEST(PartitionBucket, multipleMemoryBlock) { MemoryPool * pool = new MemoryPool(); - const uint32_t POOL_SIZE = 1024 * 1024; //1MB - const uint32_t BLOCK_SIZE = 1024; //1KB + const uint32_t POOL_SIZE = 1024 * 1024; // 1MB + const uint32_t BLOCK_SIZE = 1024; // 1KB const uint32_t PARTITION_ID = 3; pool->init(POOL_SIZE); ComparatorPtr comparator = NativeTask::get_comparator(BytesType, NULL); @@ -113,8 +111,8 @@ TEST(PartitionBucket, multipleMemoryBlock) { TEST(PartitionBucket, sort) { MemoryPool * pool = new MemoryPool(); - const uint32_t POOL_SIZE = 1024 * 1024; //1MB - const uint32_t BLOCK_SIZE = 1024; //1KB + const uint32_t POOL_SIZE = 1024 * 1024; // 1MB + const uint32_t BLOCK_SIZE = 1024; // 1KB const uint32_t PARTITION_ID = 3; pool->init(POOL_SIZE); ComparatorPtr comparator = NativeTask::get_comparator(BytesType, NULL); @@ -165,8 +163,8 @@ TEST(PartitionBucket, sort) { TEST(PartitionBucket, spill) { MemoryPool * pool = new MemoryPool(); - const uint32_t POOL_SIZE = 1024 * 1024; //1MB - const uint32_t BLOCK_SIZE = 1024; //1KB + const uint32_t POOL_SIZE = 1024 * 1024; // 1MB + const uint32_t BLOCK_SIZE = 1024; // 1KB const uint32_t PARTITION_ID = 3; pool->init(POOL_SIZE); ComparatorPtr comparator = NativeTask::get_comparator(BytesType, NULL); @@ -201,7 +199,7 @@ TEST(PartitionBucket, spill) { MockIFileWriter writer(buff, BUFF_SIZE); bucket->spill(&writer); - //check the result + // check the result KVBuffer * first = (KVBuffer *)writer.buff(); ASSERT_EQ(4, first->keyLength); ASSERT_EQ(KV_SIZE - KVBuffer::headerLength() - 4, first->valueLength); @@ -221,4 +219,4 @@ TEST(PartitionBucket, spill) { delete bucket; delete pool; } -} +} // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc index cbb636f..1a4b290 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadBuffer.cc @@ -16,10 +16,10 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "config.h" -#include "BufferStream.h" -#include "Buffers.h" +#include "lib/BufferStream.h" +#include "lib/Buffers.h" #include "test_commons.h" TEST(Buffers, AppendRead) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc index efbb8ea..f32989c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestReadWriteBuffer.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "lib/Combiner.h" #include "test_commons.h" #include <iostream> http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestTrackingCollector.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestTrackingCollector.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestTrackingCollector.cc deleted file mode 100644 index e4f7c64..0000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/lib/TestTrackingCollector.cc +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "commons.h" -#include "lib/Combiner.h" -#include "test_commons.h" -#include <iostream> - -namespace NativeTask { - -TEST(TrackingCollector, read) { - const std::string GROUP("group"); - const std::string KEY("key"); - Counter * counter = new Counter(GROUP, KEY); - Collector * collector = new Collector(); - TrackingCollector tracking(collector, counter); - tracking.collect(NULL, 0, NULL, 0); - ASSERT_EQ(1, counter->get()); - delete counter; - delete collector; -} -} /* namespace NativeTask */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc index 4ec1f48..f7332be 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc @@ -17,9 +17,9 @@ */ #include <stdarg.h> -#include "commons.h" +#include "lib/commons.h" #include "util/Random.h" -#include "FileSystem.h" +#include "lib/FileSystem.h" #include "test_commons.h" Config TestConfig = Config(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h index 48884ce..90d48ec 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h @@ -24,8 +24,8 @@ #include "util/Random.h" #include "util/StringUtil.h" #include "util/Timer.h" -#include "Buffers.h" -#include "BufferStream.h" +#include "lib/Buffers.h" +#include "lib/BufferStream.h" using std::pair; using std::vector; http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc deleted file mode 100644 index 501e005..0000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "util/Hash.h" -#include "test_commons.h" - -static uint64_t test_length(int64_t len, size_t size, size_t loopTime) { - vector<string> data; - TestConfig.setInt(GenerateLen, len); - Generate(data, size, "bytes"); - Timer t; - uint64_t ret = 0; - for (size_t m = 0; m < loopTime; m++) { - for (size_t i = 0; i < data.size(); i++) { - ret += Hash::BytesHash(data[i].c_str(), data[i].length()); - } - } - LOG("%s", t.getInterval(StringUtil::Format("Bytes%3lld", len).c_str()).c_str()); - t.reset(); - for (size_t m = 0; m < loopTime; m++) { - for (size_t i = 0; i < data.size(); i++) { - ret += Hash::CityHash(data[i].c_str(), data[i].length()); - } - } - LOG("%s", t.getInterval(StringUtil::Format(" City%3lld", len).c_str()).c_str()); - return ret; -} - -TEST(Perf, Hash) { - uint64_t ret = 0; - ret += test_length(1, 100, 4000); - ret += test_length(17, 100, 4000); - ret += test_length(64, 100, 4000); - ret += test_length(128, 100, 4000); - ret += test_length(513, 100, 4000); - fprintf(stderr, "%llu\n", (long long unsigned int)ret); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestSyncUtils.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestSyncUtils.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestSyncUtils.cc deleted file mode 100644 index 7641f9c..0000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestSyncUtils.cc +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "util/SyncUtils.h" -#include "test_commons.h" - -class TestThread : public Thread { - virtual void run() { - for (uint32_t i = 0; i < 5; i++) { - usleep(100); - LOG("sleep %d", i * 100); - } - } -}; - -TEST(SyncUtil, Thread) { - TestThread a, b, c; - a.start(); - b.start(); - c.start(); - a.join(); - b.join(); - c.join(); -} - -class TestBind { -public: - - int get() { - return 100; - } - - void foo() { - for (uint32_t i = 0; i < 2; i++) { - usleep(100); - LOG("usleep %d", i * 100); - } - } - void bar(const char * msg) { - for (uint32_t i = 0; i < 2; i++) { - usleep(100); - LOG("usleep %d %s", i * 100, msg); - } - } - -}; - -TEST(SyncUtil, ThreadBind) { - TestBind a = TestBind(); - Runnable * bind1 = BindNew(a, &TestBind::get); - Thread t = Thread(bind1); - Runnable * bind2 = BindNew(a, &TestBind::bar, "testmsg"); - Thread t2 = Thread(bind2); - t.start(); - t2.start(); - t.join(); - t2.join(); - - delete bind1; - delete bind2; -} - -//class TestParallelFor { -//protected: -// SpinLock lock; -// uint64_t aggr; -//public: -// TestParallelFor() : aggr(0) { -// } -// void add(uint64_t i) { -// lock.lock(); -// aggr += i; -// lock.unlock(); -// } -// void test(uint64_t n, size_t threadnum) { -// aggr = 0; -// ParallelFor(*this, &TestParallelFor::add, 0ULL, n, threadnum); -// ASSERT_EQ(n*(n-1)/2, aggr); -// } -//}; -// -//TEST(SyncUtil, ParallelFor) { -// TestParallelFor tpf; -// tpf.test(100000, 2); -// tpf.test(100000, 3); -// tpf.test(100000, 4); -//} - -TEST(Perf, ThreadOverhead) { - int64_t threadnum = TestConfig.getInt("thread.num", 1000); - Thread * t = new Thread[threadnum]; - Timer timer; - for (uint32_t i = 0; i < threadnum; i++) { - t[i].start(); - } - for (uint32_t i = 0; i < threadnum; i++) { - t[i].join(); - } - LOG("%lld thread %s", (long long int )threadnum, timer.getInterval("start&join").c_str()); - delete[] t; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java index 61b4356..3aa8a07 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java @@ -86,8 +86,10 @@ public class CombinerTest { Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a')); } - this.nativeoutputpath = TestConstants.NATIVETASK_COMBINER_TEST_NATIVE_OUTPUTDIR + "/nativewordcount"; - this.hadoopoutputpath = TestConstants.NATIVETASK_COMBINER_TEST_NORMAL_OUTPUTDIR + "/normalwordcount"; + this.nativeoutputpath = TestConstants.NATIVETASK_COMBINER_TEST_NATIVE_OUTPUTDIR + + "/nativewordcount"; + this.hadoopoutputpath = TestConstants.NATIVETASK_COMBINER_TEST_NORMAL_OUTPUTDIR + + "/normalwordcount"; } @AfterClass @@ -97,7 +99,8 @@ public class CombinerTest { fs.close(); } - protected static Job getJob(String jobname, Configuration inputConf, String inputpath, String outputpath) + protected static Job getJob(String jobname, Configuration inputConf, + String inputpath, String outputpath) throws Exception { final Configuration conf = new Configuration(inputConf); conf.set("fileoutputpath", outputpath); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java index d054793..a4eecbe 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java @@ -84,8 +84,10 @@ public class LargeKVCombinerTest { 1000000), IntWritable.class.getName(), Text.class.getName(), normalConf).createSequenceTestFile(inputPath, 1); - final Job normaljob = CombinerTest.getJob("normalwordcount", normalConf, inputPath, hadoopOutputPath); - final Job nativejob = CombinerTest.getJob("nativewordcount", nativeConf, inputPath, nativeOutputPath); + final Job normaljob = CombinerTest.getJob("normalwordcount", normalConf, + inputPath, hadoopOutputPath); + final Job nativejob = CombinerTest.getJob("nativewordcount", nativeConf, + inputPath, nativeOutputPath); assertTrue(nativejob.waitForCompletion(true)); @@ -93,8 +95,9 @@ public class LargeKVCombinerTest { final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath); - final String reason = "LargeKVCombinerTest failed with, min size: " + min - + ", max size: " + max + ", normal out: " + hadoopOutputPath + ", native Out: " + nativeOutputPath; + final String reason = "LargeKVCombinerTest failed with, min size: " + min + + ", max size: " + max + ", normal out: " + hadoopOutputPath + + ", native Out: " + nativeOutputPath; assertEquals(reason, true, compareRet); ResultVerifier.verifyCounters(normaljob, nativejob, true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java index 1c76bc6..c3fc02f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java @@ -55,23 +55,28 @@ public class OldAPICombinerTest { final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration(); nativeConf.addResource(TestConstants.COMBINER_CONF_PATH); final String nativeoutput = TestConstants.NATIVETASK_OLDAPI_COMBINER_TEST_NATIVE_OUTPUTPATH; - final JobConf nativeJob = getOldAPIJobconf(nativeConf, "nativeCombinerWithOldAPI", inputpath, nativeoutput); + final JobConf nativeJob = getOldAPIJobconf(nativeConf, "nativeCombinerWithOldAPI", + inputpath, nativeoutput); RunningJob nativeRunning = JobClient.runJob(nativeJob); - Counter nativeReduceGroups = nativeRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS); + Counter nativeReduceGroups = nativeRunning.getCounters().findCounter( + Task.Counter.REDUCE_INPUT_RECORDS); final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration(); normalConf.addResource(TestConstants.COMBINER_CONF_PATH); final String normaloutput = TestConstants.NATIVETASK_OLDAPI_COMBINER_TEST_NORMAL_OUTPUTPATH; - final JobConf normalJob = getOldAPIJobconf(normalConf, "normalCombinerWithOldAPI", inputpath, normaloutput); + final JobConf normalJob = getOldAPIJobconf(normalConf, "normalCombinerWithOldAPI", + inputpath, normaloutput); RunningJob normalRunning = JobClient.runJob(normalJob); - Counter normalReduceGroups = normalRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS); + Counter normalReduceGroups = normalRunning.getCounters().findCounter( + Task.Counter.REDUCE_INPUT_RECORDS); final boolean compareRet = ResultVerifier.verify(nativeoutput, normaloutput); assertEquals("file compare result: if they are the same ,then return true", true, compareRet); - assertEquals("The input reduce record count must be same", nativeReduceGroups.getValue(), normalReduceGroups.getValue()); + assertEquals("The input reduce record count must be same", + nativeReduceGroups.getValue(), normalReduceGroups.getValue()); } @Before @@ -84,7 +89,8 @@ public class OldAPICombinerTest { this.inputpath = TestConstants.NATIVETASK_COMBINER_TEST_INPUTDIR + "/wordcount"; if (!fs.exists(new Path(inputpath))) { - new TestInputFile(conf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE, 1000000), Text.class.getName(), + new TestInputFile(conf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE, 1000000), + Text.class.getName(), Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a')); } } @@ -96,7 +102,8 @@ public class OldAPICombinerTest { fs.close(); } - private static JobConf getOldAPIJobconf(Configuration configuration, String name, String input, String output) + private static JobConf getOldAPIJobconf(Configuration configuration, String name, + String input, String output) throws Exception { final JobConf jobConf = new JobConf(configuration); final FileSystem fs = FileSystem.get(configuration); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java index 490b82b..7b33672 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java @@ -44,7 +44,8 @@ public class WordCount { private final Text word = new Text(); @Override - public void map(Object key, Text value, Context context) throws IOException, InterruptedException { + public void map(Object key, Text value, Context context) + throws IOException, InterruptedException { final StringTokenizer itr = new StringTokenizer(value.toString()); while (itr.hasMoreTokens()) { word.set(itr.nextToken()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java index a11ea91..00547b6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java @@ -37,7 +37,9 @@ public class WordCountWithOldAPI { private final Text word = new Text(); @Override - public void map(Object key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) + public void map(Object key, Text value, + OutputCollector<Text, IntWritable> output, + Reporter reporter) throws IOException { final StringTokenizer itr = new StringTokenizer(value.toString()); while (itr.hasMoreTokens()) { @@ -52,8 +54,9 @@ public class WordCountWithOldAPI { private final IntWritable result = new IntWritable(); @Override - public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, - Reporter reporter) throws IOException { + public void reduce(Text key, Iterator<IntWritable> values, + OutputCollector<Text, IntWritable> output, + Reporter reporter) throws IOException { int sum = 0; while (values.hasNext()) { sum += values.next().get(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java index 1c91ff0..8198b47 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java @@ -35,12 +35,14 @@ public class CompressMapper { public static class TextCompressMapper extends Mapper<Text, Text, Text, Text> { @Override - protected void map(Text key, Text value, Context context) throws IOException, InterruptedException { + protected void map(Text key, Text value, Context context) + throws IOException, InterruptedException { context.write(key, value); } } - public static Job getCompressJob(String jobname, Configuration conf, String inputpath, String outputpath) + public static Job getCompressJob(String jobname, Configuration conf, + String inputpath, String outputpath) throws Exception { Job job = new Job(conf, jobname + "-CompressMapperJob"); job.setJarByClass(CompressMapper.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java index 7b337b7..81d632d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestCombineHandler.java @@ -41,13 +41,13 @@ public class TestCombineHandler extends TestCase { @Override public void setUp() throws IOException { - this.nativeHandler = Mockito.mock(INativeHandler.class); this.pusher = Mockito.mock(BufferPusher.class); this.puller = Mockito.mock(BufferPuller.class); this.combinerRunner = Mockito.mock(CombinerRunner.class); - Mockito.when(nativeHandler.getInputBuffer()).thenReturn(new InputBuffer(BufferType.HEAP_BUFFER, 100)); + Mockito.when(nativeHandler.getInputBuffer()).thenReturn( + new InputBuffer(BufferType.HEAP_BUFFER, 100)); } public void testCommandDispatcherSetting() throws IOException { @@ -62,7 +62,8 @@ public class TestCombineHandler extends TestCase { handler.close(); handler.close(); - Mockito.verify(combinerRunner, Mockito.times(1)).combine(Matchers.eq(puller), Matchers.eq(pusher)); + Mockito.verify(combinerRunner, Mockito.times(1)) + .combine(Matchers.eq(puller), Matchers.eq(pusher)); Mockito.verify(pusher, Mockito.times(1)).close(); Mockito.verify(puller, Mockito.times(1)).close(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java index 6b9edc8..6c15360 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/handlers/TestNativeCollectorOnlyHandler.java @@ -65,7 +65,8 @@ public class TestNativeCollectorOnlyHandler extends TestCase { null, null); - Mockito.when(nativeHandler.getInputBuffer()).thenReturn(new InputBuffer(BufferType.HEAP_BUFFER, 100)); + Mockito.when(nativeHandler.getInputBuffer()).thenReturn( + new InputBuffer(BufferType.HEAP_BUFFER, 100)); } @Override @@ -91,7 +92,8 @@ public class TestNativeCollectorOnlyHandler extends TestCase { public void testGetCombiner() throws IOException { this.handler = new NativeCollectorOnlyHandler(taskContext, nativeHandler, pusher, combiner); Mockito.when(combiner.getId()).thenReturn(100L); - final ReadWriteBuffer result = handler.onCall(NativeCollectorOnlyHandler.GET_COMBINE_HANDLER, null); + final ReadWriteBuffer result = handler.onCall( + NativeCollectorOnlyHandler.GET_COMBINE_HANDLER, null); Assert.assertEquals(100L, result.readLong()); } @@ -109,13 +111,16 @@ public class TestNativeCollectorOnlyHandler extends TestCase { final String expectedOutputIndexPath = LOCAL_DIR + "/output/file.out.index"; final String expectedSpillPath = LOCAL_DIR + "/output/spill0.out"; - final String outputPath = handler.onCall(NativeCollectorOnlyHandler.GET_OUTPUT_PATH, null).readString(); + final String outputPath = handler.onCall( + NativeCollectorOnlyHandler.GET_OUTPUT_PATH, null).readString(); Assert.assertEquals(expectedOutputPath, outputPath); - final String outputIndexPath = handler.onCall(NativeCollectorOnlyHandler.GET_OUTPUT_INDEX_PATH, null).readString(); + final String outputIndexPath = handler.onCall( + NativeCollectorOnlyHandler.GET_OUTPUT_INDEX_PATH, null).readString(); Assert.assertEquals(expectedOutputIndexPath, outputIndexPath); - final String spillPath = handler.onCall(NativeCollectorOnlyHandler.GET_SPILL_PATH, null).readString(); + final String spillPath = handler.onCall( + NativeCollectorOnlyHandler.GET_SPILL_PATH, null).readString(); Assert.assertEquals(expectedSpillPath, spillPath); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java index ce23780..2a72d05 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java @@ -32,7 +32,8 @@ public class HashSumReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, I DataOutputStream dos = new DataOutputStream(os); @Override - public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) throws IOException, InterruptedException { + public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) + throws IOException, InterruptedException { int hashSum = 0; for (final VTYPE val : values) { if (val instanceof Writable) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java index 9e7bb5e..0a4c955 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java @@ -45,13 +45,15 @@ public class KVJob { public static class ValueMapper<KTYPE, VTYPE> extends Mapper<KTYPE, VTYPE, KTYPE, VTYPE> { @Override - public void map(KTYPE key, VTYPE value, Context context) throws IOException, InterruptedException { + public void map(KTYPE key, VTYPE value, Context context) + throws IOException, InterruptedException { context.write(key, value); } } public static class KVMReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, VTYPE> { - public void reduce(KTYPE key, VTYPE value, Context context) throws IOException, InterruptedException { + public void reduce(KTYPE key, VTYPE value, Context context) + throws IOException, InterruptedException { context.write(key, value); } } @@ -59,7 +61,8 @@ public class KVJob { public static class KVReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, VTYPE> { @Override - public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) throws IOException, InterruptedException { + public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) + throws IOException, InterruptedException { long resultlong = 0;// 8 bytes match BytesFactory.fromBytes function final CRC32 crc32 = new CRC32(); for (final VTYPE val : values) { @@ -68,12 +71,14 @@ public class KVJob { resultlong += crc32.getValue(); } final VTYPE V = null; - context.write(key, (VTYPE) BytesFactory.newObject(Longs.toByteArray(resultlong), V.getClass().getName())); + context.write(key, (VTYPE) BytesFactory.newObject(Longs.toByteArray(resultlong), + V.getClass().getName())); } } - public KVJob(String jobname, Configuration conf, Class<?> keyclass, Class<?> valueclass, String inputpath, - String outputpath) throws Exception { + public KVJob(String jobname, Configuration conf, + Class<?> keyclass, Class<?> valueclass, + String inputpath, String outputpath) throws Exception { job = new Job(conf, jobname); job.setJarByClass(KVJob.class); job.setMapperClass(KVJob.ValueMapper.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java index 4c77f4c..49331ce 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java @@ -89,7 +89,8 @@ public class TestInputFile { map.put(IntWritable.class.getName(), new KVSizeScope(4, 4)); } - public TestInputFile(int filesize, String keytype, String valuetype, Configuration conf) throws Exception { + public TestInputFile(int filesize, String keytype, String valuetype, + Configuration conf) throws Exception { this.filesize = filesize; this.databuf = new byte[DATABUFSIZE]; this.keyClsName = keytype;
