Repository: parquet-cpp Updated Branches: refs/heads/master dc0fc7d49 -> 674dbb39c
http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/buffer.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/buffer.h b/src/parquet/util/buffer.h index 0cd973a..c0f263f 100644 --- a/src/parquet/util/buffer.h +++ b/src/parquet/util/buffer.h @@ -36,49 +36,35 @@ namespace parquet { // class instance class Buffer : public std::enable_shared_from_this<Buffer> { public: - Buffer(const uint8_t* data, int64_t size) : - data_(data), - size_(size) {} + Buffer(const uint8_t* data, int64_t size) : data_(data), size_(size) {} // An offset into data that is owned by another buffer, but we want to be // able to retain a valid pointer to it even after other shared_ptr's to the // parent buffer have been destroyed Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size); - std::shared_ptr<Buffer> get_shared_ptr() { - return shared_from_this(); - } + std::shared_ptr<Buffer> get_shared_ptr() { return shared_from_this(); } // Return true if both buffers are the same size and contain the same bytes // up to the number of compared bytes bool Equals(const Buffer& other, int64_t nbytes) const { - return this == &other || - (size_ >= nbytes && other.size_ >= nbytes && - !memcmp(data_, other.data_, nbytes)); + return this == &other || (size_ >= nbytes && other.size_ >= nbytes && + !memcmp(data_, other.data_, nbytes)); } bool Equals(const Buffer& other) const { - return this == &other || - (size_ == other.size_ && !memcmp(data_, other.data_, size_)); + return this == &other || (size_ == other.size_ && !memcmp(data_, other.data_, size_)); } - const uint8_t* data() const { - return data_; - } + const uint8_t* data() const { return data_; } - int64_t size() const { - return size_; - } + int64_t size() const { return size_; } // Returns true if this Buffer is referencing memory (possibly) owned by some // other buffer - bool is_shared() const { - return static_cast<bool>(parent_); - } + bool is_shared() const { return static_cast<bool>(parent_); } - const std::shared_ptr<Buffer> parent() const { - return parent_; - } + const std::shared_ptr<Buffer> parent() const { return parent_; } protected: const uint8_t* data_; @@ -94,22 +80,17 @@ class Buffer : public std::enable_shared_from_this<Buffer> { // A Buffer whose contents can be mutated. May or may not own its data. class MutableBuffer : public Buffer { public: - MutableBuffer(uint8_t* data, int64_t size) : - Buffer(data, size) { + MutableBuffer(uint8_t* data, int64_t size) : Buffer(data, size) { mutable_data_ = data; } - uint8_t* mutable_data() { - return mutable_data_; - } + uint8_t* mutable_data() { return mutable_data_; } // Get a read-only view of this buffer std::shared_ptr<Buffer> GetImmutableView(); protected: - MutableBuffer() : - Buffer(nullptr, 0), - mutable_data_(nullptr) {} + MutableBuffer() : Buffer(nullptr, 0), mutable_data_(nullptr) {} uint8_t* mutable_data_; }; @@ -119,8 +100,8 @@ class ResizableBuffer : public MutableBuffer { virtual void Resize(int64_t new_size) = 0; protected: - ResizableBuffer(uint8_t* data, int64_t size) : - MutableBuffer(data, size), capacity_(size) {} + ResizableBuffer(uint8_t* data, int64_t size) + : MutableBuffer(data, size), capacity_(size) {} int64_t capacity_; }; @@ -129,8 +110,8 @@ class ResizableBuffer : public MutableBuffer { // garbage-collected class OwnedMutableBuffer : public ResizableBuffer { public: - explicit OwnedMutableBuffer(int64_t size = 0, - MemoryAllocator* allocator = default_allocator()); + explicit OwnedMutableBuffer( + int64_t size = 0, MemoryAllocator* allocator = default_allocator()); virtual ~OwnedMutableBuffer(); void Resize(int64_t new_size) override; void Reserve(int64_t new_capacity); @@ -151,9 +132,7 @@ class Vector { void Reserve(int64_t new_capacity); void Assign(int64_t size, const T val); void Swap(Vector<T>& v); - inline T& operator[](int64_t i) { - return data_[i]; - } + inline T& operator[](int64_t i) { return data_[i]; } private: std::unique_ptr<OwnedMutableBuffer> buffer_; @@ -164,6 +143,6 @@ class Vector { DISALLOW_COPY_AND_ASSIGN(Vector); }; -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_BUFFER_H +#endif // PARQUET_UTIL_BUFFER_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/compiler-util.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/compiler-util.h b/src/parquet/util/compiler-util.h index 9048ba1..3f2c373 100644 --- a/src/parquet/util/compiler-util.h +++ b/src/parquet/util/compiler-util.h @@ -36,25 +36,23 @@ #define PREFETCH(addr) __builtin_prefetch(addr) -//macros to disable padding -//these macros are portable across different compilers and platforms +// macros to disable padding +// these macros are portable across different compilers and platforms //[https://github.com/google/flatbuffers/blob/master/include/flatbuffers/flatbuffers.h#L1355] #if defined(_MSC_VER) - #define MANUALLY_ALIGNED_STRUCT(alignment) \ - __pragma(pack(1)); \ - struct __declspec(align(alignment)) - #define STRUCT_END(name, size) \ - __pragma(pack()); \ - static_assert(sizeof(name) == size, "compiler breaks packing rules") +#define MANUALLY_ALIGNED_STRUCT(alignment) \ + __pragma(pack(1)); \ + struct __declspec(align(alignment)) +#define STRUCT_END(name, size) \ + __pragma(pack()); \ + static_assert(sizeof(name) == size, "compiler breaks packing rules") #elif defined(__GNUC__) || defined(__clang__) - #define MANUALLY_ALIGNED_STRUCT(alignment) \ - _Pragma("pack(1)") \ - struct __attribute__((aligned(alignment))) - #define STRUCT_END(name, size) \ - _Pragma("pack()") \ - static_assert(sizeof(name) == size, "compiler breaks packing rules") +#define MANUALLY_ALIGNED_STRUCT(alignment) \ + _Pragma("pack(1)") struct __attribute__((aligned(alignment))) +#define STRUCT_END(name, size) \ + _Pragma("pack()") static_assert(sizeof(name) == size, "compiler breaks packing rules") #else - #error Unknown compiler, please define structure alignment macros +#error Unknown compiler, please define structure alignment macros #endif -#endif // PARQUET_UTIL_COMPILER_UTIL_H +#endif // PARQUET_UTIL_COMPILER_UTIL_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/cpu-info.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/cpu-info.cc b/src/parquet/util/cpu-info.cc index 836152f..1b1bc00 100644 --- a/src/parquet/util/cpu-info.cc +++ b/src/parquet/util/cpu-info.cc @@ -52,16 +52,14 @@ int64_t CpuInfo::original_hardware_flags_; int64_t CpuInfo::cache_sizes_[L3_CACHE + 1]; int64_t CpuInfo::cycles_per_ms_; int CpuInfo::num_cores_ = 1; -string CpuInfo::model_name_ = "unknown"; // NOLINT +string CpuInfo::model_name_ = "unknown"; // NOLINT static struct { string name; int64_t flag; } flag_mappings[] = { - { "ssse3", CpuInfo::SSSE3 }, - { "sse4_1", CpuInfo::SSE4_1 }, - { "sse4_2", CpuInfo::SSE4_2 }, - { "popcnt", CpuInfo::POPCNT }, + {"ssse3", CpuInfo::SSSE3}, {"sse4_1", CpuInfo::SSE4_1}, {"sse4_2", CpuInfo::SSE4_2}, + {"popcnt", CpuInfo::POPCNT}, }; static const int64_t num_flags = sizeof(flag_mappings) / sizeof(flag_mappings[0]); @@ -72,9 +70,7 @@ static const int64_t num_flags = sizeof(flag_mappings) / sizeof(flag_mappings[0] int64_t ParseCPUFlags(const string& values) { int64_t flags = 0; for (int i = 0; i < num_flags; ++i) { - if (contains(values, flag_mappings[i].name)) { - flags |= flag_mappings[i].flag; - } + if (contains(values, flag_mappings[i].name)) { flags |= flag_mappings[i].flag; } } return flags; } @@ -167,4 +163,4 @@ void CpuInfo::EnableFeature(int64_t flag, bool enable) { } } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/cpu-info.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/cpu-info.h b/src/parquet/util/cpu-info.h index dd951e5..a2fd5de 100644 --- a/src/parquet/util/cpu-info.h +++ b/src/parquet/util/cpu-info.h @@ -34,10 +34,10 @@ namespace parquet { /// /sys/devices) class CpuInfo { public: - static const int64_t SSSE3 = (1 << 1); - static const int64_t SSE4_1 = (1 << 2); - static const int64_t SSE4_2 = (1 << 3); - static const int64_t POPCNT = (1 << 4); + static const int64_t SSSE3 = (1 << 1); + static const int64_t SSE4_1 = (1 << 2); + static const int64_t SSE4_2 = (1 << 3); + static const int64_t POPCNT = (1 << 4); /// Cache enums for L1 (data), L2 and L3 enum CacheLevel { @@ -93,9 +93,7 @@ class CpuInfo { return model_name_; } - static bool initialized() { - return initialized_; - } + static bool initialized() { return initialized_; } private: static bool initialized_; @@ -104,9 +102,9 @@ class CpuInfo { static int64_t cache_sizes_[L3_CACHE + 1]; static int64_t cycles_per_ms_; static int num_cores_; - static std::string model_name_; // NOLINT + static std::string model_name_; // NOLINT }; -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_CPU_INFO_H +#endif // PARQUET_UTIL_CPU_INFO_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/hash-util.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/hash-util.h b/src/parquet/util/hash-util.h index e3f376f..96701da 100644 --- a/src/parquet/util/hash-util.h +++ b/src/parquet/util/hash-util.h @@ -104,7 +104,7 @@ class HashUtil { const uint64_t* p = reinterpret_cast<const uint64_t*>(v); hash = SSE4_crc32_u64(hash, *p); ++p; - hash = SSE4_crc32_u32(hash, *reinterpret_cast<const uint32_t *>(p)); + hash = SSE4_crc32_u32(hash, *reinterpret_cast<const uint32_t*>(p)); hash = (hash << 16) | (hash >> 16); return hash; } @@ -141,14 +141,21 @@ class HashUtil { const uint8_t* data2 = reinterpret_cast<const uint8_t*>(data); switch (len & 7) { - case 7: h ^= uint64_t(data2[6]) << 48; - case 6: h ^= uint64_t(data2[5]) << 40; - case 5: h ^= uint64_t(data2[4]) << 32; - case 4: h ^= uint64_t(data2[3]) << 24; - case 3: h ^= uint64_t(data2[2]) << 16; - case 2: h ^= uint64_t(data2[1]) << 8; - case 1: h ^= uint64_t(data2[0]); - h *= MURMUR_PRIME; + case 7: + h ^= uint64_t(data2[6]) << 48; + case 6: + h ^= uint64_t(data2[5]) << 40; + case 5: + h ^= uint64_t(data2[4]) << 32; + case 4: + h ^= uint64_t(data2[3]) << 24; + case 3: + h ^= uint64_t(data2[2]) << 16; + case 2: + h ^= uint64_t(data2[1]) << 8; + case 1: + h ^= uint64_t(data2[0]); + h *= MURMUR_PRIME; } h ^= h >> MURMUR_R; @@ -158,8 +165,8 @@ class HashUtil { } /// default values recommended by http://isthe.com/chongo/tech/comp/fnv/ - static const uint32_t FNV_PRIME = 0x01000193; // 16777619 - static const uint32_t FNV_SEED = 0x811C9DC5; // 2166136261 + static const uint32_t FNV_PRIME = 0x01000193; // 16777619 + static const uint32_t FNV_SEED = 0x811C9DC5; // 2166136261 static const uint64_t FNV64_PRIME = 1099511628211UL; static const uint64_t FNV64_SEED = 14695981039346656037UL; @@ -246,6 +253,6 @@ class HashUtil { } }; -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_HASH_UTIL_H +#endif // PARQUET_UTIL_HASH_UTIL_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/input-output-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/input-output-test.cc b/src/parquet/util/input-output-test.cc index 3e15183..9db2bdd 100644 --- a/src/parquet/util/input-output-test.cc +++ b/src/parquet/util/input-output-test.cc @@ -90,9 +90,7 @@ class TestFileReaders : public ::testing::Test { public: void SetUp() { test_path_ = "parquet-input-output-test.txt"; - if (file_exists(test_path_)) { - std::remove(test_path_.c_str()); - } + if (file_exists(test_path_)) { std::remove(test_path_.c_str()); } test_data_ = "testingdata"; std::ofstream stream; @@ -101,14 +99,10 @@ class TestFileReaders : public ::testing::Test { filesize_ = test_data_.size(); } - void TearDown() { - DeleteTestFile(); - } + void TearDown() { DeleteTestFile(); } void DeleteTestFile() { - if (file_exists(test_path_)) { - std::remove(test_path_.c_str()); - } + if (file_exists(test_path_)) { std::remove(test_path_.c_str()); } } protected: @@ -153,4 +147,4 @@ TYPED_TEST(TestFileReaders, BadSeek) { ASSERT_THROW(this->source.Seek(this->filesize_ + 1), ParquetException); } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/input.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/input.cc b/src/parquet/util/input.cc index 2a0eda3..8eb3956 100644 --- a/src/parquet/util/input.cc +++ b/src/parquet/util/input.cc @@ -92,9 +92,7 @@ void LocalFileSource::Seek(int64_t pos) { int64_t LocalFileSource::Tell() const { int64_t position = ftell(file_); - if (position < 0) { - throw ParquetException("ftell failed, did the file disappear?"); - } + if (position < 0) { throw ParquetException("ftell failed, did the file disappear?"); } return position; } @@ -111,9 +109,7 @@ std::shared_ptr<Buffer> LocalFileSource::Read(int64_t nbytes) { result->Resize(nbytes); int64_t bytes_read = Read(nbytes, result->mutable_data()); - if (bytes_read < nbytes) { - result->Resize(bytes_read); - } + if (bytes_read < nbytes) { result->Resize(bytes_read); } return result; } // ---------------------------------------------------------------------- @@ -125,12 +121,10 @@ MemoryMapSource::~MemoryMapSource() { void MemoryMapSource::Open(const std::string& path) { LocalFileSource::Open(path); - data_ = reinterpret_cast<uint8_t*>(mmap(nullptr, size_, PROT_READ, - MAP_SHARED, fileno(file_), 0)); - if (data_ == nullptr) { - throw ParquetException("Memory mapping file failed"); - } - pos_ = 0; + data_ = reinterpret_cast<uint8_t*>( + mmap(nullptr, size_, PROT_READ, MAP_SHARED, fileno(file_), 0)); + if (data_ == nullptr) { throw ParquetException("Memory mapping file failed"); } + pos_ = 0; } void MemoryMapSource::Close() { @@ -139,9 +133,7 @@ void MemoryMapSource::Close() { } void MemoryMapSource::CloseFile() { - if (data_ != nullptr) { - munmap(data_, size_); - } + if (data_ != nullptr) { munmap(data_, size_); } LocalFileSource::CloseFile(); } @@ -177,10 +169,8 @@ std::shared_ptr<Buffer> MemoryMapSource::Read(int64_t nbytes) { // ---------------------------------------------------------------------- // BufferReader -BufferReader::BufferReader(const std::shared_ptr<Buffer>& buffer) : - buffer_(buffer), - data_(buffer->data()), - pos_(0) { +BufferReader::BufferReader(const std::shared_ptr<Buffer>& buffer) + : buffer_(buffer), data_(buffer->data()), pos_(0) { size_ = buffer->size(); } @@ -191,8 +181,7 @@ int64_t BufferReader::Tell() const { void BufferReader::Seek(int64_t pos) { if (pos < 0 || pos >= size_) { std::stringstream ss; - ss << "Cannot seek to " << pos - << "File is length " << size_; + ss << "Cannot seek to " << pos << "File is length " << size_; throw ParquetException(ss.str()); } pos_ = pos; @@ -215,8 +204,8 @@ std::shared_ptr<Buffer> BufferReader::Read(int64_t nbytes) { // ---------------------------------------------------------------------- // InMemoryInputStream -InMemoryInputStream::InMemoryInputStream(const std::shared_ptr<Buffer>& buffer) : - buffer_(buffer), offset_(0) { +InMemoryInputStream::InMemoryInputStream(const std::shared_ptr<Buffer>& buffer) + : buffer_(buffer), offset_(0) { len_ = buffer_->size(); } @@ -235,4 +224,4 @@ void InMemoryInputStream::Advance(int64_t num_bytes) { offset_ += num_bytes; } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/input.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/input.h b/src/parquet/util/input.h index dae3e91..04bbb34 100644 --- a/src/parquet/util/input.h +++ b/src/parquet/util/input.h @@ -53,11 +53,10 @@ class RandomAccessSource { int64_t size_; }; - class LocalFileSource : public RandomAccessSource { public: - explicit LocalFileSource(MemoryAllocator* allocator = default_allocator()) : - file_(nullptr), is_open_(false), allocator_(allocator) {} + explicit LocalFileSource(MemoryAllocator* allocator = default_allocator()) + : file_(nullptr), is_open_(false), allocator_(allocator) {} virtual ~LocalFileSource(); @@ -72,8 +71,8 @@ class LocalFileSource : public RandomAccessSource { virtual std::shared_ptr<Buffer> Read(int64_t nbytes); - bool is_open() const { return is_open_;} - const std::string& path() const { return path_;} + bool is_open() const { return is_open_; } + const std::string& path() const { return path_; } // Return the integer file descriptor int file_descriptor() const; @@ -90,8 +89,8 @@ class LocalFileSource : public RandomAccessSource { class MemoryMapSource : public LocalFileSource { public: - explicit MemoryMapSource(MemoryAllocator* allocator = default_allocator()) : - LocalFileSource(allocator), data_(nullptr), pos_(0) {} + explicit MemoryMapSource(MemoryAllocator* allocator = default_allocator()) + : LocalFileSource(allocator), data_(nullptr), pos_(0) {} virtual ~MemoryMapSource(); @@ -130,9 +129,7 @@ class BufferReader : public RandomAccessSource { virtual std::shared_ptr<Buffer> Read(int64_t nbytes); protected: - const uint8_t* Head() { - return data_ + pos_; - } + const uint8_t* Head() { return data_ + pos_; } std::shared_ptr<Buffer> buffer_; const uint8_t* data_; @@ -183,6 +180,6 @@ class InMemoryInputStream : public InputStream { int64_t offset_; }; -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_INPUT_H +#endif // PARQUET_UTIL_INPUT_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/logging.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/logging.h b/src/parquet/util/logging.h index 8aa750b..8d3e88a 100644 --- a/src/parquet/util/logging.h +++ b/src/parquet/util/logging.h @@ -43,13 +43,27 @@ namespace parquet { #ifdef NDEBUG #define PARQUET_DFATAL PARQUET_WARNING -#define DCHECK(condition) while (false) parquet::internal::NullLog() -#define DCHECK_EQ(val1, val2) while (false) parquet::internal::NullLog() -#define DCHECK_NE(val1, val2) while (false) parquet::internal::NullLog() -#define DCHECK_LE(val1, val2) while (false) parquet::internal::NullLog() -#define DCHECK_LT(val1, val2) while (false) parquet::internal::NullLog() -#define DCHECK_GE(val1, val2) while (false) parquet::internal::NullLog() -#define DCHECK_GT(val1, val2) while (false) parquet::internal::NullLog() +#define DCHECK(condition) \ + while (false) \ + parquet::internal::NullLog() +#define DCHECK_EQ(val1, val2) \ + while (false) \ + parquet::internal::NullLog() +#define DCHECK_NE(val1, val2) \ + while (false) \ + parquet::internal::NullLog() +#define DCHECK_LE(val1, val2) \ + while (false) \ + parquet::internal::NullLog() +#define DCHECK_LT(val1, val2) \ + while (false) \ + parquet::internal::NullLog() +#define DCHECK_GE(val1, val2) \ + while (false) \ + parquet::internal::NullLog() +#define DCHECK_GT(val1, val2) \ + while (false) \ + parquet::internal::NullLog() #else #define PARQUET_DFATAL PARQUET_FATAL @@ -62,13 +76,13 @@ namespace parquet { #define DCHECK_GE(val1, val2) PARQUET_CHECK((val1) >= (val2)) #define DCHECK_GT(val1, val2) PARQUET_CHECK((val1) > (val2)) -#endif // NDEBUG +#endif // NDEBUG namespace internal { class NullLog { public: - template<class T> + template <class T> NullLog& operator<<(const T& t) { return *this; } @@ -76,21 +90,16 @@ class NullLog { class CerrLog { public: - CerrLog(int severity) // NOLINT(runtime/explicit) - : severity_(severity), - has_logged_(false) { - } + CerrLog(int severity) // NOLINT(runtime/explicit) + : severity_(severity), + has_logged_(false) {} ~CerrLog() { - if (has_logged_) { - std::cerr << std::endl; - } - if (severity_ == PARQUET_FATAL) { - exit(1); - } + if (has_logged_) { std::cerr << std::endl; } + if (severity_ == PARQUET_FATAL) { exit(1); } } - template<class T> + template <class T> CerrLog& operator<<(const T& t) { has_logged_ = true; std::cerr << t; @@ -102,8 +111,8 @@ class CerrLog { bool has_logged_; }; -} // namespace internal +} // namespace internal -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_LOGGING_H +#endif // PARQUET_UTIL_LOGGING_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/macros.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/macros.h b/src/parquet/util/macros.h index d221173..8f704b2 100644 --- a/src/parquet/util/macros.h +++ b/src/parquet/util/macros.h @@ -21,8 +21,8 @@ // Useful macros from elsewhere // From Google gutil -#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ - TypeName(const TypeName&) = delete; \ +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&) = delete; \ void operator=(const TypeName&) = delete // ---------------------------------------------------------------------- @@ -46,7 +46,7 @@ // // Can call MyClass::MyMethod() here. // } -#define FRIEND_TEST(test_case_name, test_name)\ -friend class test_case_name##_##test_name##_Test +#define FRIEND_TEST(test_case_name, test_name) \ + friend class test_case_name##_##test_name##_Test -#endif // PARQUET_UTIL_MACROS_H +#endif // PARQUET_UTIL_MACROS_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/mem-allocator-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/mem-allocator-test.cc b/src/parquet/util/mem-allocator-test.cc index 2e86e19..336d3b4 100644 --- a/src/parquet/util/mem-allocator-test.cc +++ b/src/parquet/util/mem-allocator-test.cc @@ -64,4 +64,4 @@ TEST(TestAllocator, TotalMax) { ASSERT_EQ(110, allocator.MaxMemory()); } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/mem-allocator.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/mem-allocator.cc b/src/parquet/util/mem-allocator.cc index dc2df2b..4d42bc4 100644 --- a/src/parquet/util/mem-allocator.cc +++ b/src/parquet/util/mem-allocator.cc @@ -26,18 +26,12 @@ namespace parquet { MemoryAllocator::~MemoryAllocator() {} uint8_t* TrackingAllocator::Malloc(int64_t size) { - if (0 == size) { - return nullptr; - } + if (0 == size) { return nullptr; } uint8_t* p = static_cast<uint8_t*>(std::malloc(size)); - if (!p) { - throw ParquetException("OOM: memory allocation failed"); - } + if (!p) { throw ParquetException("OOM: memory allocation failed"); } total_memory_ += size; - if (total_memory_ > max_memory_) { - max_memory_ = total_memory_; - } + if (total_memory_ > max_memory_) { max_memory_ = total_memory_; } return p; } @@ -58,4 +52,4 @@ MemoryAllocator* default_allocator() { return &default_allocator; } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/mem-allocator.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/mem-allocator.h b/src/parquet/util/mem-allocator.h index 612a050..eb68f02 100644 --- a/src/parquet/util/mem-allocator.h +++ b/src/parquet/util/mem-allocator.h @@ -34,7 +34,7 @@ class MemoryAllocator { MemoryAllocator* default_allocator(); -class TrackingAllocator: public MemoryAllocator { +class TrackingAllocator : public MemoryAllocator { public: TrackingAllocator() : total_memory_(0), max_memory_(0) {} virtual ~TrackingAllocator(); @@ -42,19 +42,15 @@ class TrackingAllocator: public MemoryAllocator { uint8_t* Malloc(int64_t size) override; void Free(uint8_t* p, int64_t size) override; - int64_t TotalMemory() { - return total_memory_; - } + int64_t TotalMemory() { return total_memory_; } - int64_t MaxMemory() { - return max_memory_; - } + int64_t MaxMemory() { return max_memory_; } private: int64_t total_memory_; int64_t max_memory_; }; -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_MEMORY_POOL_H +#endif // PARQUET_UTIL_MEMORY_POOL_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/mem-pool-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/mem-pool-test.cc b/src/parquet/util/mem-pool-test.cc index b5151e5..7e1443a 100644 --- a/src/parquet/util/mem-pool-test.cc +++ b/src/parquet/util/mem-pool-test.cc @@ -137,16 +137,16 @@ TEST(MemPoolTest, Basic) { // free chunks. TEST(MemPoolTest, Keep) { MemPool p; - p.Allocate(4*1024); - p.Allocate(8*1024); - p.Allocate(16*1024); + p.Allocate(4 * 1024); + p.Allocate(8 * 1024); + p.Allocate(16 * 1024); EXPECT_EQ((4 + 8 + 16) * 1024, p.total_allocated_bytes()); EXPECT_EQ((4 + 8 + 16) * 1024, p.GetTotalChunkSizes()); p.Clear(); EXPECT_EQ(0, p.total_allocated_bytes()); EXPECT_EQ((4 + 8 + 16) * 1024, p.GetTotalChunkSizes()); - p.Allocate(1*1024); - p.Allocate(4*1024); + p.Allocate(1 * 1024); + p.Allocate(4 * 1024); EXPECT_EQ((1 + 4) * 1024, p.total_allocated_bytes()); EXPECT_EQ((4 + 8 + 16) * 1024, p.GetTotalChunkSizes()); @@ -244,4 +244,4 @@ TEST(MemPoolTest, FragmentationOverhead) { p.FreeAll(); } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/mem-pool.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/mem-pool.cc b/src/parquet/util/mem-pool.cc index 73817da..b211bee 100644 --- a/src/parquet/util/mem-pool.cc +++ b/src/parquet/util/mem-pool.cc @@ -35,18 +35,15 @@ const int MemPool::INITIAL_CHUNK_SIZE; const int MemPool::MAX_CHUNK_SIZE; MemPool::MemPool(MemoryAllocator* allocator) - : current_chunk_idx_(-1), - next_chunk_size_(INITIAL_CHUNK_SIZE), - total_allocated_bytes_(0), - peak_allocated_bytes_(0), - total_reserved_bytes_(0), - allocator_(allocator) {} + : current_chunk_idx_(-1), + next_chunk_size_(INITIAL_CHUNK_SIZE), + total_allocated_bytes_(0), + peak_allocated_bytes_(0), + total_reserved_bytes_(0), + allocator_(allocator) {} MemPool::ChunkInfo::ChunkInfo(int64_t size, uint8_t* buf) - : data(buf), - size(size), - allocated_bytes(0) { -} + : data(buf), size(size), allocated_bytes(0) {} MemPool::~MemPool() { int64_t total_bytes_released = 0; @@ -86,7 +83,7 @@ bool MemPool::FindChunk(int64_t min_size) { int first_free_idx = current_chunk_idx_ + 1; // (cast size() to signed int in order to avoid everything else being cast to // unsigned long, in particular -1) - while (++current_chunk_idx_ < static_cast<int>(chunks_.size())) { + while (++current_chunk_idx_ < static_cast<int>(chunks_.size())) { // we found a free chunk DCHECK_EQ(chunks_[current_chunk_idx_].allocated_bytes, 0); @@ -127,8 +124,8 @@ bool MemPool::FindChunk(int64_t min_size) { total_reserved_bytes_ += chunk_size; // Don't increment the chunk size until the allocation succeeds: if an attempted // large allocation fails we don't want to increase the chunk size further. - next_chunk_size_ = static_cast<int>(std::min<int64_t>( - chunk_size * 2, MAX_CHUNK_SIZE)); + next_chunk_size_ = + static_cast<int>(std::min<int64_t>(chunk_size * 2, MAX_CHUNK_SIZE)); } DCHECK_LT(current_chunk_idx_, static_cast<int>(chunks_.size())); @@ -188,16 +185,13 @@ std::string MemPool::DebugString() { char str[16]; out << "MemPool(#chunks=" << chunks_.size() << " ["; for (size_t i = 0; i < chunks_.size(); ++i) { - sprintf(str, "0x%lx=", reinterpret_cast<size_t>(chunks_[i].data)); // NOLINT - out << (i > 0 ? " " : "") - << str - << chunks_[i].size - << "/" << chunks_[i].allocated_bytes; + sprintf(str, "0x%lx=", reinterpret_cast<size_t>(chunks_[i].data)); // NOLINT + out << (i > 0 ? " " : "") << str << chunks_[i].size << "/" + << chunks_[i].allocated_bytes; } out << "] current_chunk=" << current_chunk_idx_ << " total_sizes=" << GetTotalChunkSizes() - << " total_alloc=" << total_allocated_bytes_ - << ")"; + << " total_alloc=" << total_allocated_bytes_ << ")"; return out.str(); } @@ -232,4 +226,4 @@ bool MemPool::CheckIntegrity(bool current_chunk_empty) { return true; } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/mem-pool.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/mem-pool.h b/src/parquet/util/mem-pool.h index 1022eb8..b64ee29 100644 --- a/src/parquet/util/mem-pool.h +++ b/src/parquet/util/mem-pool.h @@ -85,9 +85,7 @@ class MemPool { /// Allocates 8-byte aligned section of memory of 'size' bytes at the end /// of the the current chunk. Creates a new chunk if there aren't any chunks /// with enough capacity. - uint8_t* Allocate(int size) { - return Allocate<false>(size); - } + uint8_t* Allocate(int size) { return Allocate<false>(size); } /// Returns 'byte_size' to the current chunk back to the mem pool. This can /// only be used to return either all or part of the previous allocation returned @@ -131,18 +129,15 @@ class MemPool { static const int MAX_CHUNK_SIZE = 1024 * 1024; struct ChunkInfo { - uint8_t* data; // Owned by the ChunkInfo. - int64_t size; // in bytes + uint8_t* data; // Owned by the ChunkInfo. + int64_t size; // in bytes /// bytes allocated via Allocate() in this chunk int64_t allocated_bytes; explicit ChunkInfo(int64_t size, uint8_t* buf); - ChunkInfo() - : data(NULL), - size(0), - allocated_bytes(0) {} + ChunkInfo() : data(NULL), size(0), allocated_bytes(0) {} }; /// chunk from which we served the last Allocate() call; @@ -189,9 +184,9 @@ class MemPool { if (size == 0) return NULL; int64_t num_bytes = BitUtil::RoundUp(size, 8); - if (current_chunk_idx_ == -1 - || num_bytes + chunks_[current_chunk_idx_].allocated_bytes - > chunks_[current_chunk_idx_].size) { + if (current_chunk_idx_ == -1 || + num_bytes + chunks_[current_chunk_idx_].allocated_bytes > + chunks_[current_chunk_idx_].size) { // If we couldn't allocate a new chunk, return NULL. if (UNLIKELY(!FindChunk(num_bytes))) return NULL; } @@ -206,6 +201,6 @@ class MemPool { } }; -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_MEM_POOL_H +#endif // PARQUET_UTIL_MEM_POOL_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/output.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/output.cc b/src/parquet/util/output.cc index 704a13b..1d27355 100644 --- a/src/parquet/util/output.cc +++ b/src/parquet/util/output.cc @@ -28,11 +28,10 @@ namespace parquet { // ---------------------------------------------------------------------- // In-memory output stream -InMemoryOutputStream::InMemoryOutputStream(int64_t initial_capacity, - MemoryAllocator* allocator) : size_(0), capacity_(initial_capacity) { - if (initial_capacity == 0) { - initial_capacity = IN_MEMORY_DEFAULT_CAPACITY; - } +InMemoryOutputStream::InMemoryOutputStream( + int64_t initial_capacity, MemoryAllocator* allocator) + : size_(0), capacity_(initial_capacity) { + if (initial_capacity == 0) { initial_capacity = IN_MEMORY_DEFAULT_CAPACITY; } buffer_.reset(new OwnedMutableBuffer(initial_capacity, allocator)); } @@ -64,4 +63,4 @@ std::shared_ptr<Buffer> InMemoryOutputStream::GetBuffer() { return result; } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/output.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/output.h b/src/parquet/util/output.h index 4cfa162..472a9cc 100644 --- a/src/parquet/util/output.h +++ b/src/parquet/util/output.h @@ -74,6 +74,6 @@ class InMemoryOutputStream : public OutputStream { DISALLOW_COPY_AND_ASSIGN(InMemoryOutputStream); }; -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_OUTPUT_H +#endif // PARQUET_UTIL_OUTPUT_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/rle-encoding.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/rle-encoding.h b/src/parquet/util/rle-encoding.h index 52388da..20c9621 100644 --- a/src/parquet/util/rle-encoding.h +++ b/src/parquet/util/rle-encoding.h @@ -85,11 +85,11 @@ class RleDecoder { /// Create a decoder object. buffer/buffer_len is the decoded data. /// bit_width is the width of each value (before encoding). RleDecoder(const uint8_t* buffer, int buffer_len, int bit_width) - : bit_reader_(buffer, buffer_len), - bit_width_(bit_width), - current_value_(0), - repeat_count_(0), - literal_count_(0) { + : bit_reader_(buffer, buffer_len), + bit_width_(bit_width), + current_value_(0), + repeat_count_(0), + literal_count_(0) { DCHECK_GE(bit_width_, 0); DCHECK_LE(bit_width_, 64); } @@ -107,7 +107,7 @@ class RleDecoder { } /// Gets the next value. Returns false if there are no more. - template<typename T> + template <typename T> bool Get(T* val); protected: @@ -121,7 +121,7 @@ class RleDecoder { private: /// Fills literal_count_ and repeat_count_ with next values. Returns false if there /// are no more. - template<typename T> + template <typename T> bool NextCounts(); }; @@ -140,8 +140,7 @@ class RleEncoder { /// based on the bit_width, which can determine a storage optimal choice. /// TODO: allow 0 bit_width (and have dict encoder use it) RleEncoder(uint8_t* buffer, int buffer_len, int bit_width) - : bit_width_(bit_width), - bit_writer_(buffer, buffer_len) { + : bit_width_(bit_width), bit_writer_(buffer, buffer_len) { DCHECK_GE(bit_width_, 0); DCHECK_LE(bit_width_, 64); max_run_byte_size_ = MinBufferSize(bit_width); @@ -154,8 +153,8 @@ class RleEncoder { /// It is not valid to pass a buffer less than this length. static int MinBufferSize(int bit_width) { /// 1 indicator byte and MAX_VALUES_PER_LITERAL_RUN 'bit_width' values. - int max_literal_run_size = 1 + - BitUtil::Ceil(MAX_VALUES_PER_LITERAL_RUN * bit_width, 8); + int max_literal_run_size = + 1 + BitUtil::Ceil(MAX_VALUES_PER_LITERAL_RUN * bit_width, 8); /// Up to MAX_VLQ_BYTE_LEN indicator and a single 'bit_width' value. int max_repeated_run_size = BitReader::MAX_VLQ_BYTE_LEN + BitUtil::Ceil(bit_width, 8); return std::max(max_literal_run_size, max_repeated_run_size); @@ -248,7 +247,7 @@ class RleEncoder { uint8_t* literal_indicator_byte_; }; -template<typename T> +template <typename T> inline bool RleDecoder::Get(T* val) { DCHECK_GE(bit_width_, 0); if (UNLIKELY(literal_count_ == 0 && repeat_count_ == 0)) { @@ -268,7 +267,7 @@ inline bool RleDecoder::Get(T* val) { return true; } -template<typename T> +template <typename T> bool RleDecoder::NextCounts() { // Read the next run's indicator int, it could be a literal or repeated run. // The int is encoded as a vlq-encoded value. @@ -399,16 +398,16 @@ inline void RleEncoder::FlushBufferedValues(bool done) { inline int RleEncoder::Flush() { if (literal_count_ > 0 || repeat_count_ > 0 || num_buffered_values_ > 0) { - bool all_repeat = literal_count_ == 0 && - (repeat_count_ == num_buffered_values_ || num_buffered_values_ == 0); + bool all_repeat = literal_count_ == 0 && (repeat_count_ == num_buffered_values_ || + num_buffered_values_ == 0); // There is something pending, figure out if it's a repeated or literal run if (repeat_count_ > 0 && all_repeat) { FlushRepeatedRun(); - } else { + } else { DCHECK_EQ(literal_count_ % 8, 0); // Buffer the last group of literals to 8 by padding with 0s. for (; num_buffered_values_ != 0 && num_buffered_values_ < 8; - ++num_buffered_values_) { + ++num_buffered_values_) { buffered_values_[num_buffered_values_] = 0; } literal_count_ += num_buffered_values_; @@ -441,6 +440,6 @@ inline void RleEncoder::Clear() { bit_writer_.Clear(); } -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_RLE_ENCODING_H +#endif // PARQUET_UTIL_RLE_ENCODING_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/rle-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/rle-test.cc b/src/parquet/util/rle-test.cc index 64fd57b..d781137 100644 --- a/src/parquet/util/rle-test.cc +++ b/src/parquet/util/rle-test.cc @@ -104,7 +104,7 @@ TEST(BitArray, TestBool) { // Writes 'num_vals' values with width 'bit_width' and reads them back. void TestBitArrayValues(int bit_width, int num_vals) { const int len = BitUtil::Ceil(bit_width * num_vals, 8); - const uint64_t mod = bit_width == 64? 1 : 1LL << bit_width; + const uint64_t mod = bit_width == 64 ? 1 : 1LL << bit_width; uint8_t buffer[len]; BitWriter writer(buffer, len); @@ -176,8 +176,8 @@ TEST(BitArray, TestMixed) { // expected_encoding != NULL, also validates that the encoded buffer is // exactly 'expected_encoding'. // if expected_len is not -1, it will validate the encoded size is correct. -void ValidateRle(const vector<int>& values, int bit_width, - uint8_t* expected_encoding, int expected_len) { +void ValidateRle(const vector<int>& values, int bit_width, uint8_t* expected_encoding, + int expected_len) { const int len = 64 * 1024; uint8_t buffer[len]; EXPECT_LE(expected_len, len); @@ -189,9 +189,7 @@ void ValidateRle(const vector<int>& values, int bit_width, } int encoded_len = encoder.Flush(); - if (expected_len != -1) { - EXPECT_EQ(encoded_len, expected_len); - } + if (expected_len != -1) { EXPECT_EQ(encoded_len, expected_len); } if (expected_encoding != NULL) { EXPECT_TRUE(memcmp(buffer, expected_encoding, expected_len) == 0); } @@ -214,9 +212,7 @@ bool CheckRoundTrip(const vector<int>& values, int bit_width) { RleEncoder encoder(buffer, len, bit_width); for (size_t i = 0; i < values.size(); ++i) { bool result = encoder.Put(values[i]); - if (!result) { - return false; - } + if (!result) { return false; } } int encoded_len = encoder.Flush(); int out; @@ -224,9 +220,7 @@ bool CheckRoundTrip(const vector<int>& values, int bit_width) { RleDecoder decoder(buffer, encoded_len, bit_width); for (size_t i = 0; i < values.size(); ++i) { EXPECT_TRUE(decoder.Get(&out)); - if (values[i] != out) { - return false; - } + if (values[i] != out) { return false; } } return true; } @@ -264,11 +258,11 @@ TEST(Rle, SpecificSequences) { } int num_groups = BitUtil::Ceil(100, 8); expected_buffer[0] = (num_groups << 1) | 1; - for (int i = 1; i <= 100/8; ++i) { + for (int i = 1; i <= 100 / 8; ++i) { expected_buffer[i] = BOOST_BINARY(1 0 1 0 1 0 1 0); } // Values for the last 4 0 and 1's. The upper 4 bits should be padded to 0. - expected_buffer[100/8 + 1] = BOOST_BINARY(0 0 0 0 1 0 1 0); + expected_buffer[100 / 8 + 1] = BOOST_BINARY(0 0 0 0 1 0 1 0); // num_groups and expected_buffer only valid for bit width = 1 ValidateRle(values, 1, expected_buffer, 1 + num_groups); @@ -301,13 +295,13 @@ TEST(Rle, TestValues) { TEST(Rle, BitWidthZeroRepeated) { uint8_t buffer[1]; const int num_values = 15; - buffer[0] = num_values << 1; // repeated indicator byte + buffer[0] = num_values << 1; // repeated indicator byte RleDecoder decoder(buffer, sizeof(buffer), 0); uint8_t val; for (int i = 0; i < num_values; ++i) { bool result = decoder.Get(&val); EXPECT_TRUE(result); - EXPECT_EQ(val, 0); // can only encode 0s with bit width 0 + EXPECT_EQ(val, 0); // can only encode 0s with bit width 0 } EXPECT_FALSE(decoder.Get(&val)); } @@ -315,14 +309,14 @@ TEST(Rle, BitWidthZeroRepeated) { TEST(Rle, BitWidthZeroLiteral) { uint8_t buffer[1]; const int num_groups = 4; - buffer[0] = num_groups << 1 | 1; // literal indicator byte + buffer[0] = num_groups << 1 | 1; // literal indicator byte RleDecoder decoder = RleDecoder(buffer, sizeof(buffer), 0); const int num_values = num_groups * 8; uint8_t val; for (int i = 0; i < num_values; ++i) { bool result = decoder.Get(&val); EXPECT_TRUE(result); - EXPECT_EQ(val, 0); // can only encode 0s with bit width 0 + EXPECT_EQ(val, 0); // can only encode 0s with bit width 0 } EXPECT_FALSE(decoder.Get(&val)); } @@ -331,7 +325,8 @@ TEST(Rle, BitWidthZeroLiteral) { // group but flush before finishing. TEST(BitRle, Flush) { vector<int> values; - for (int i = 0; i < 16; ++i) values.push_back(1); + for (int i = 0; i < 16; ++i) + values.push_back(1); values.push_back(0); ValidateRle(values, 1, NULL, -1); values.push_back(1); @@ -363,9 +358,7 @@ TEST(BitRle, Random) { for (int i = 0; i < ngroups; ++i) { int group_size = dist(gen); - if (group_size > max_group_size) { - group_size = 1; - } + if (group_size > max_group_size) { group_size = 1; } for (int i = 0; i < group_size; ++i) { values.push_back(parity); } @@ -437,4 +430,4 @@ TEST(BitRle, Overflow) { } } -} // namespace parquet +} // namespace parquet http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/sse-util.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/sse-util.h b/src/parquet/util/sse-util.h index 6288d1d..653c171 100644 --- a/src/parquet/util/sse-util.h +++ b/src/parquet/util/sse-util.h @@ -27,51 +27,36 @@ namespace parquet { - /// This class contains constants useful for text processing with SSE4.2 intrinsics. namespace SSEUtil { - /// Number of characters that fit in 64/128 bit register. SSE provides instructions - /// for loading 64 or 128 bits into a register at a time. - static const int CHARS_PER_64_BIT_REGISTER = 8; - static const int CHARS_PER_128_BIT_REGISTER = 16; - - /// SSE4.2 adds instructions for text processing. The instructions have a control - /// byte that determines some of functionality of the instruction. (Equivalent to - /// GCC's _SIDD_CMP_EQUAL_ANY, etc). - static const int PCMPSTR_EQUAL_ANY = 0x00; // strchr - static const int PCMPSTR_EQUAL_EACH = 0x08; // strcmp - static const int PCMPSTR_UBYTE_OPS = 0x00; // unsigned char (8-bits, rather than 16) - static const int PCMPSTR_NEG_POLARITY = 0x10; // see Intel SDM chapter 4.1.4. - - /// In this mode, SSE text processing functions will return a mask of all the - /// characters that matched. - static const int STRCHR_MODE = PCMPSTR_EQUAL_ANY | PCMPSTR_UBYTE_OPS; - - /// In this mode, SSE text processing functions will return the number of - /// bytes that match consecutively from the beginning. - static const int STRCMP_MODE = PCMPSTR_EQUAL_EACH | PCMPSTR_UBYTE_OPS | - PCMPSTR_NEG_POLARITY; - - /// Precomputed mask values up to 16 bits. - static const int SSE_BITMASK[CHARS_PER_128_BIT_REGISTER] = { - 1 << 0, - 1 << 1, - 1 << 2, - 1 << 3, - 1 << 4, - 1 << 5, - 1 << 6, - 1 << 7, - 1 << 8, - 1 << 9, - 1 << 10, - 1 << 11, - 1 << 12, - 1 << 13, - 1 << 14, - 1 << 15, - }; -} // namespace SSEUtil +/// Number of characters that fit in 64/128 bit register. SSE provides instructions +/// for loading 64 or 128 bits into a register at a time. +static const int CHARS_PER_64_BIT_REGISTER = 8; +static const int CHARS_PER_128_BIT_REGISTER = 16; + +/// SSE4.2 adds instructions for text processing. The instructions have a control +/// byte that determines some of functionality of the instruction. (Equivalent to +/// GCC's _SIDD_CMP_EQUAL_ANY, etc). +static const int PCMPSTR_EQUAL_ANY = 0x00; // strchr +static const int PCMPSTR_EQUAL_EACH = 0x08; // strcmp +static const int PCMPSTR_UBYTE_OPS = 0x00; // unsigned char (8-bits, rather than 16) +static const int PCMPSTR_NEG_POLARITY = 0x10; // see Intel SDM chapter 4.1.4. + +/// In this mode, SSE text processing functions will return a mask of all the +/// characters that matched. +static const int STRCHR_MODE = PCMPSTR_EQUAL_ANY | PCMPSTR_UBYTE_OPS; + +/// In this mode, SSE text processing functions will return the number of +/// bytes that match consecutively from the beginning. +static const int STRCMP_MODE = + PCMPSTR_EQUAL_EACH | PCMPSTR_UBYTE_OPS | PCMPSTR_NEG_POLARITY; + +/// Precomputed mask values up to 16 bits. +static const int SSE_BITMASK[CHARS_PER_128_BIT_REGISTER] = { + 1 << 0, 1 << 1, 1 << 2, 1 << 3, 1 << 4, 1 << 5, 1 << 6, 1 << 7, 1 << 8, 1 << 9, + 1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15, +}; +} // namespace SSEUtil #ifdef PARQUET_USE_SSE @@ -88,29 +73,35 @@ namespace SSEUtil { /// The PCMPxSTRy instructions require that the control byte 'mode' be encoded as an /// immediate. So, those need to be always inlined in order to always propagate the /// mode constant into the inline asm. -#define SSE_ALWAYS_INLINE inline __attribute__ ((__always_inline__)) +#define SSE_ALWAYS_INLINE inline __attribute__((__always_inline__)) -template<int MODE> +template <int MODE> static inline __m128i SSE4_cmpestrm(__m128i str1, int len1, __m128i str2, int len2) { #ifdef __clang__ /// Use asm reg rather than Yz output constraint to workaround LLVM bug 13199 - /// clang doesn't support Y-prefixed asm constraints. register volatile __m128i result asm("xmm0"); - __asm__ volatile ("pcmpestrm %5, %2, %1" - : "=x"(result) : "x"(str1), "xm"(str2), "a"(len1), "d"(len2), "i"(MODE) : "cc"); + __asm__ volatile("pcmpestrm %5, %2, %1" + : "=x"(result) + : "x"(str1), "xm"(str2), "a"(len1), "d"(len2), "i"(MODE) + : "cc"); #else __m128i result; - __asm__ volatile ("pcmpestrm %5, %2, %1" - : "=Yz"(result) : "x"(str1), "xm"(str2), "a"(len1), "d"(len2), "i"(MODE) : "cc"); + __asm__ volatile("pcmpestrm %5, %2, %1" + : "=Yz"(result) + : "x"(str1), "xm"(str2), "a"(len1), "d"(len2), "i"(MODE) + : "cc"); #endif return result; } -template<int MODE> +template <int MODE> static inline int SSE4_cmpestri(__m128i str1, int len1, __m128i str2, int len2) { int result; __asm__("pcmpestri %5, %2, %1" - : "=c"(result) : "x"(str1), "xm"(str2), "a"(len1), "d"(len2), "i"(MODE) : "cc"); + : "=c"(result) + : "x"(str1), "xm"(str2), "a"(len1), "d"(len2), "i"(MODE) + : "cc"); return result; } @@ -143,7 +134,7 @@ static inline int64_t POPCNT_popcnt_u64(uint64_t a) { #undef SSE_ALWAYS_INLINE -#elif defined(__SSE4_2__) // IR_COMPILE for SSE 4.2. +#elif defined(__SSE4_2__) // IR_COMPILE for SSE 4.2. /// When cross-compiling to IR, we cannot use inline asm because LLVM JIT does not /// support it. However, the cross-compiled IR is compiled twice: with and without /// -msse4.2. When -msse4.2 is enabled in the cross-compile, we can just use the @@ -151,15 +142,13 @@ static inline int64_t POPCNT_popcnt_u64(uint64_t a) { #include <smmintrin.h> -template<int MODE> -static inline __m128i SSE4_cmpestrm( - __m128i str1, int len1, __m128i str2, int len2) { +template <int MODE> +static inline __m128i SSE4_cmpestrm(__m128i str1, int len1, __m128i str2, int len2) { return _mm_cmpestrm(str1, len1, str2, len2, MODE); } -template<int MODE> -static inline int SSE4_cmpestri( - __m128i str1, int len1, __m128i str2, int len2) { +template <int MODE> +static inline int SSE4_cmpestri(__m128i str1, int len1, __m128i str2, int len2) { return _mm_cmpestri(str1, len1, str2, len2, MODE); } @@ -175,13 +164,13 @@ static inline int SSE4_cmpestri( /// support SSE 4.2. However, because the caller isn't allowed to call these routines /// on CPUs that lack SSE 4.2 anyway, we can implement stubs for this case. -template<int MODE> +template <int MODE> static inline __m128i SSE4_cmpestrm(__m128i str1, int len1, __m128i str2, int len2) { DCHECK(false) << "CPU doesn't support SSE 4.2"; - return (__m128i) { 0 }; // NOLINT + return (__m128i){0}; // NOLINT } -template<int MODE> +template <int MODE> static inline int SSE4_cmpestri(__m128i str1, int len1, __m128i str2, int len2) { DCHECK(false) << "CPU doesn't support SSE 4.2"; return 0; @@ -212,7 +201,7 @@ static inline int64_t POPCNT_popcnt_u64(uint64_t a) { return 0; } -#endif // IR_COMPILE +#endif // IR_COMPILE #else @@ -241,8 +230,8 @@ static inline int64_t POPCNT_popcnt_u64(uint64_t a) { return 0; } -#endif // PARQUET_USE_SSE +#endif // PARQUET_USE_SSE -} // namespace parquet +} // namespace parquet -#endif // PARQUET_UTIL_SSE_UTIL_H +#endif // PARQUET_UTIL_SSE_UTIL_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/stopwatch.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/stopwatch.h b/src/parquet/util/stopwatch.h index 612a00c..b940d8c 100644 --- a/src/parquet/util/stopwatch.h +++ b/src/parquet/util/stopwatch.h @@ -28,26 +28,23 @@ namespace parquet { class StopWatch { public: - StopWatch() { - } + StopWatch() {} - void Start() { - gettimeofday(&start_time, 0); - } + void Start() { gettimeofday(&start_time, 0); } // Returns time in nanoseconds. uint64_t Stop() { struct timeval t_time; gettimeofday(&t_time, 0); - return (1000L * 1000L * 1000L * (t_time.tv_sec - start_time.tv_sec) - + (t_time.tv_usec - start_time.tv_usec)); + return (1000L * 1000L * 1000L * (t_time.tv_sec - start_time.tv_sec) + + (t_time.tv_usec - start_time.tv_usec)); } private: - struct timeval start_time; + struct timeval start_time; }; -} // namespace parquet +} // namespace parquet #endif http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/test-common.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/test-common.h b/src/parquet/util/test-common.h index 019af8e..edadb53 100644 --- a/src/parquet/util/test-common.h +++ b/src/parquet/util/test-common.h @@ -31,13 +31,11 @@ namespace parquet { namespace test { -typedef ::testing::Types<BooleanType, Int32Type, Int64Type, Int96Type, - FloatType, DoubleType, ByteArrayType, - FLBAType> ParquetTypes; +typedef ::testing::Types<BooleanType, Int32Type, Int64Type, Int96Type, FloatType, + DoubleType, ByteArrayType, FLBAType> ParquetTypes; template <typename T> -static inline void assert_vector_equal(const vector<T>& left, - const vector<T>& right) { +static inline void assert_vector_equal(const vector<T>& left, const vector<T>& right) { ASSERT_EQ(left.size(), right.size()); for (size_t i = 0; i < left.size(); ++i) { @@ -47,15 +45,11 @@ static inline void assert_vector_equal(const vector<T>& left, template <typename T> static inline bool vector_equal(const vector<T>& left, const vector<T>& right) { - if (left.size() != right.size()) { - return false; - } + if (left.size() != right.size()) { return false; } for (size_t i = 0; i < left.size(); ++i) { if (left[i] != right[i]) { - std::cerr << "index " << i - << " left was " << left[i] - << " right was " << right[i] + std::cerr << "index " << i << " left was " << left[i] << " right was " << right[i] << std::endl; return false; } @@ -66,9 +60,7 @@ static inline bool vector_equal(const vector<T>& left, const vector<T>& right) { template <typename T> static vector<T> slice(const vector<T>& values, int start, int end) { - if (end < start) { - return vector<T>(0); - } + if (end < start) { return vector<T>(0); } vector<T> out(end - start); for (int i = start; i < end; ++i) { @@ -137,8 +129,8 @@ void random_numbers(int n, uint32_t seed, float min_value, float max_value, floa } template <> -void random_numbers(int n, uint32_t seed, double min_value, double max_value, - double* out) { +void random_numbers( + int n, uint32_t seed, double min_value, double max_value, double* out) { std::mt19937 gen(seed); std::uniform_real_distribution<double> d(min_value, max_value); for (int i = 0; i < n; ++i) { @@ -146,8 +138,8 @@ void random_numbers(int n, uint32_t seed, double min_value, double max_value, } } -void random_Int96_numbers(int n, uint32_t seed, int32_t min_value, int32_t max_value, - Int96* out) { +void random_Int96_numbers( + int n, uint32_t seed, int32_t min_value, int32_t max_value, Int96* out) { std::mt19937 gen(seed); std::uniform_int_distribution<int32_t> d(min_value, max_value); for (int i = 0; i < n; ++i) { @@ -157,8 +149,7 @@ void random_Int96_numbers(int n, uint32_t seed, int32_t min_value, int32_t max_v } } -void random_fixed_byte_array(int n, uint32_t seed, uint8_t *buf, int len, - FLBA* out) { +void random_fixed_byte_array(int n, uint32_t seed, uint8_t* buf, int len, FLBA* out) { std::mt19937 gen(seed); std::uniform_int_distribution<int> d(0, 255); for (int i = 0; i < n; ++i) { @@ -170,8 +161,8 @@ void random_fixed_byte_array(int n, uint32_t seed, uint8_t *buf, int len, } } -void random_byte_array(int n, uint32_t seed, uint8_t *buf, - ByteArray* out, int min_size, int max_size) { +void random_byte_array( + int n, uint32_t seed, uint8_t* buf, ByteArray* out, int min_size, int max_size) { std::mt19937 gen(seed); std::uniform_int_distribution<int> d1(min_size, max_size); std::uniform_int_distribution<int> d2(0, 255); @@ -186,12 +177,11 @@ void random_byte_array(int n, uint32_t seed, uint8_t *buf, } } -void random_byte_array(int n, uint32_t seed, uint8_t *buf, - ByteArray* out, int max_size) { +void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int max_size) { random_byte_array(n, seed, buf, out, 0, max_size); } -} // namespace test -} // namespace parquet +} // namespace test +} // namespace parquet -#endif // PARQUET_UTIL_TEST_COMMON_H +#endif // PARQUET_UTIL_TEST_COMMON_H http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/674dbb39/src/parquet/util/test_main.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/test_main.cc b/src/parquet/util/test_main.cc index 00139f3..6fb7c05 100644 --- a/src/parquet/util/test_main.cc +++ b/src/parquet/util/test_main.cc @@ -17,7 +17,7 @@ #include <gtest/gtest.h> -int main(int argc, char **argv) { +int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); int ret = RUN_ALL_TESTS();
