Repository: parquet-cpp Updated Branches: refs/heads/master cd14a3d0f -> 5c1d9e94b
PARQUET-728: Incorporate upstream Arrow API changes I bumped the thirdparty git hash to Arrow trunk. Pardon the diff noise from clang format. Also resolves PARQUET-729. Author: Wes McKinney <[email protected]> Closes #167 from wesm/PARQUET-728 and squashes the following commits: 222e72c [Wes McKinney] Write multiple columns in file-serialize-test. Fix PARQUET-729 ef56871 [Wes McKinney] Incorporate upstream Arrow API changes, clang format with LLVM 3.8.0 Project: http://git-wip-us.apache.org/repos/asf/parquet-cpp/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-cpp/commit/5c1d9e94 Tree: http://git-wip-us.apache.org/repos/asf/parquet-cpp/tree/5c1d9e94 Diff: http://git-wip-us.apache.org/repos/asf/parquet-cpp/diff/5c1d9e94 Branch: refs/heads/master Commit: 5c1d9e94b8409d1a2b31491e68f92b4207e48646 Parents: cd14a3d Author: Wes McKinney <[email protected]> Authored: Sun Sep 25 19:22:48 2016 -0400 Committer: Wes McKinney <[email protected]> Committed: Sun Sep 25 19:22:48 2016 -0400 ---------------------------------------------------------------------- CMakeLists.txt | 2 +- benchmarks/decode_benchmark.cc | 6 +- cmake_modules/FindArrow.cmake | 4 +- src/parquet/arrow/CMakeLists.txt | 7 ++- src/parquet/arrow/arrow-io-test.cc | 54 +---------------- src/parquet/arrow/arrow-reader-writer-test.cc | 9 +-- src/parquet/arrow/io.cc | 2 +- src/parquet/arrow/io.h | 4 +- src/parquet/arrow/reader.cc | 15 ++--- src/parquet/arrow/reader.h | 2 +- src/parquet/arrow/utils.h | 16 ++--- src/parquet/arrow/writer.cc | 23 +++---- src/parquet/column/column-io-benchmark.cc | 4 +- src/parquet/column/column-reader-test.cc | 2 +- src/parquet/column/column-writer-test.cc | 35 ++++++----- src/parquet/column/levels-test.cc | 4 +- src/parquet/column/levels.h | 2 +- src/parquet/column/properties.h | 2 +- src/parquet/column/scanner-test.cc | 7 ++- src/parquet/column/scanner.h | 2 +- src/parquet/column/test-specialization.h | 57 ++++++++++-------- src/parquet/column/test-util.h | 4 +- src/parquet/compression/codec-test.cc | 2 +- src/parquet/compression/snappy-codec.cc | 2 +- .../encodings/delta-byte-array-encoding.h | 2 +- src/parquet/encodings/encoding-test.cc | 9 +-- src/parquet/exception.cc | 4 ++ src/parquet/exception.h | 1 + src/parquet/file/file-deserialize-test.cc | 2 +- src/parquet/file/file-metadata-test.cc | 2 +- src/parquet/file/file-serialize-test.cc | 63 +++++++++++++------- src/parquet/file/metadata.h | 2 +- src/parquet/file/reader-internal.cc | 6 +- src/parquet/file/reader.cc | 2 +- src/parquet/file/reader.h | 2 +- src/parquet/file/writer-internal.cc | 4 +- src/parquet/reader-test.cc | 10 ++-- src/parquet/schema/descriptor.h | 6 +- src/parquet/schema/schema-descriptor-test.cc | 2 +- src/parquet/thrift/util.h | 10 ++-- src/parquet/util/bit-stream-utils.h | 4 +- src/parquet/util/bit-util-test.cc | 8 +-- src/parquet/util/buffer-builder.h | 2 +- src/parquet/util/buffer-test.cc | 4 +- src/parquet/util/cpu-info.cc | 2 +- src/parquet/util/input.cc | 2 +- src/parquet/util/mem-pool-test.cc | 4 +- src/parquet/util/mem-pool.h | 4 +- src/parquet/util/rle-encoding.h | 4 +- src/parquet/util/rle-test.cc | 4 +- src/parquet/util/stopwatch.h | 2 +- src/parquet/util/test-common.h | 3 +- thirdparty/versions.sh | 2 +- tools/parquet-scan.cc | 2 +- tools/parquet_reader.cc | 2 +- 55 files changed, 216 insertions(+), 227 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index 42b10ee..3cac3e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -416,7 +416,7 @@ if (UNIX) add_custom_target(lint ${BUILD_SUPPORT_DIR}/cpplint.py --verbose=2 --linelength=90 - --filter=-whitespace/comments,-readability/todo,-build/header_guard,-runtime/references,-readability/check,-build/c++11 + --filter=-whitespace/comments,-readability/todo,-build/header_guard,-runtime/references,-readability/check,-build/c++11,-build/include_order `find ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/tools ${CMAKE_CURRENT_SOURCE_DIR}/benchmarks -name \\*.cc -or -name \\*.h | sed -e '/parquet\\/thrift/g'`) endif (UNIX) http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/benchmarks/decode_benchmark.cc ---------------------------------------------------------------------- diff --git a/benchmarks/decode_benchmark.cc b/benchmarks/decode_benchmark.cc index d3748ca..7659234 100644 --- a/benchmarks/decode_benchmark.cc +++ b/benchmarks/decode_benchmark.cc @@ -15,16 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include <stdio.h> #include <iostream> #include <random> +#include <stdio.h> #include "parquet/compression/codec.h" -#include "parquet/encodings/plain-encoding.h" -#include "parquet/encodings/dictionary-encoding.h" #include "parquet/encodings/delta-bit-pack-encoding.h" #include "parquet/encodings/delta-byte-array-encoding.h" #include "parquet/encodings/delta-length-byte-array-encoding.h" +#include "parquet/encodings/dictionary-encoding.h" +#include "parquet/encodings/plain-encoding.h" #include "parquet/util/stopwatch.h" /** http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/cmake_modules/FindArrow.cmake ---------------------------------------------------------------------- diff --git a/cmake_modules/FindArrow.cmake b/cmake_modules/FindArrow.cmake index 91d0e71..eec5d66 100644 --- a/cmake_modules/FindArrow.cmake +++ b/cmake_modules/FindArrow.cmake @@ -53,10 +53,10 @@ if (ARROW_INCLUDE_DIR AND ARROW_LIB_PATH) set(ARROW_IO_LIB_NAME libarrow_io) set(ARROW_LIBS ${ARROW_SEARCH_LIB_PATH}) - set(ARROW_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_LIB_NAME}.a) + set(ARROW_STATIC_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}.a) set(ARROW_SHARED_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) - set(ARROW_IO_STATIC_LIB ${ARROW_SEARCH_LIB_PATH}/${ARROW_IO_LIB_NAME}.a) + set(ARROW_IO_STATIC_LIB ${ARROW_LIBS}/${ARROW_IO_LIB_NAME}.a) set(ARROW_IO_SHARED_LIB ${ARROW_LIBS}/${ARROW_IO_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) if (NOT Arrow_FIND_QUIETLY) message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}") http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/arrow/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/CMakeLists.txt b/src/parquet/arrow/CMakeLists.txt index 5d923a7..f36abe6 100644 --- a/src/parquet/arrow/CMakeLists.txt +++ b/src/parquet/arrow/CMakeLists.txt @@ -48,6 +48,10 @@ if (PARQUET_BUILD_SHARED) BUILD_WITH_INSTALL_RPATH ON INSTALL_NAME_DIR "@rpath") endif() + + install(TARGETS parquet_arrow_shared + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib) endif() if (PARQUET_BUILD_STATIC) @@ -58,7 +62,7 @@ if (PARQUET_BUILD_STATIC) OUTPUT_NAME "parquet_arrow") target_link_libraries(parquet_arrow_static arrow_static - arrow_static + arrow_io parquet_static) install(TARGETS parquet_arrow_static ARCHIVE DESTINATION lib @@ -87,4 +91,3 @@ install(FILES utils.h writer.h DESTINATION include/parquet/arrow) - http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/arrow/arrow-io-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/arrow-io-test.cc b/src/parquet/arrow/arrow-io-test.cc index 377fb97..344dd13 100644 --- a/src/parquet/arrow/arrow-io-test.cc +++ b/src/parquet/arrow/arrow-io-test.cc @@ -22,6 +22,7 @@ #include "gtest/gtest.h" +#include "arrow/io/memory.h" #include "arrow/test-util.h" #include "arrow/util/memory-pool.h" #include "arrow/util/status.h" @@ -33,8 +34,7 @@ using arrow::default_memory_pool; using arrow::MemoryPool; using arrow::Status; -// To assist with readability -using ArrowROFile = arrow::io::RandomAccessFile; +using ArrowBufferReader = arrow::io::BufferReader; namespace parquet { namespace arrow { @@ -103,61 +103,13 @@ TEST(TestParquetAllocator, CustomPool) { // ---------------------------------------------------------------------- // Read source tests -class BufferReader : public ArrowROFile { - public: - BufferReader(const uint8_t* buffer, int buffer_size) - : buffer_(buffer), buffer_size_(buffer_size), position_(0) {} - - Status Close() override { - // no-op - return Status::OK(); - } - - Status Tell(int64_t* position) override { - *position = position_; - return Status::OK(); - } - - Status ReadAt( - int64_t position, int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override { - RETURN_NOT_OK(Seek(position)); - return Read(nbytes, bytes_read, buffer); - } - - Status Read(int64_t nbytes, int64_t* bytes_read, uint8_t* buffer) override { - memcpy(buffer, buffer_ + position_, nbytes); - *bytes_read = std::min(nbytes, buffer_size_ - position_); - position_ += *bytes_read; - return Status::OK(); - } - - Status GetSize(int64_t* size) override { - *size = buffer_size_; - return Status::OK(); - } - - Status Seek(int64_t position) override { - if (position < 0 || position >= buffer_size_) { - return Status::IOError("position out of bounds"); - } - - position_ = position; - return Status::OK(); - } - - private: - const uint8_t* buffer_; - int buffer_size_; - int64_t position_; -}; - TEST(TestParquetReadSource, Basics) { std::string data = "this is the data"; auto data_buffer = reinterpret_cast<const uint8_t*>(data.c_str()); ParquetAllocator allocator(default_memory_pool()); - auto file = std::make_shared<BufferReader>(data_buffer, data.size()); + auto file = std::make_shared<ArrowBufferReader>(data_buffer, data.size()); auto source = std::make_shared<ParquetReadSource>(&allocator); ASSERT_OK(source->Open(file)); http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/arrow/arrow-reader-writer-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/arrow-reader-writer-test.cc b/src/parquet/arrow/arrow-reader-writer-test.cc index e4e9efa..b1f1c52 100644 --- a/src/parquet/arrow/arrow-reader-writer-test.cc +++ b/src/parquet/arrow/arrow-reader-writer-test.cc @@ -20,8 +20,8 @@ #include "parquet/api/reader.h" #include "parquet/api/writer.h" -#include "parquet/arrow/test-util.h" #include "parquet/arrow/reader.h" +#include "parquet/arrow/test-util.h" #include "parquet/arrow/writer.h" #include "arrow/test-util.h" @@ -258,7 +258,8 @@ class TestParquetIO : public ::testing::Test { typedef ::testing::Types<::arrow::BooleanType, ::arrow::UInt8Type, ::arrow::Int8Type, ::arrow::UInt16Type, ::arrow::Int16Type, ::arrow::Int32Type, ::arrow::UInt64Type, ::arrow::Int64Type, ::arrow::TimestampType, ::arrow::FloatType, ::arrow::DoubleType, - ::arrow::StringType> TestTypes; + ::arrow::StringType> + TestTypes; TYPED_TEST_CASE(TestParquetIO, TestTypes); @@ -476,8 +477,8 @@ class TestPrimitiveParquetIO : public TestParquetIO<TestType> { typedef ::testing::Types<::arrow::BooleanType, ::arrow::UInt8Type, ::arrow::Int8Type, ::arrow::UInt16Type, ::arrow::Int16Type, ::arrow::UInt32Type, ::arrow::Int32Type, - ::arrow::UInt64Type, ::arrow::Int64Type, ::arrow::FloatType, - ::arrow::DoubleType> PrimitiveTestTypes; + ::arrow::UInt64Type, ::arrow::Int64Type, ::arrow::FloatType, ::arrow::DoubleType> + PrimitiveTestTypes; TYPED_TEST_CASE(TestPrimitiveParquetIO, PrimitiveTestTypes); http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/arrow/io.cc ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/io.cc b/src/parquet/arrow/io.cc index 8e2645a..8f3aa3e 100644 --- a/src/parquet/arrow/io.cc +++ b/src/parquet/arrow/io.cc @@ -29,7 +29,7 @@ using arrow::Status; using arrow::MemoryPool; // To assist with readability -using ArrowROFile = arrow::io::RandomAccessFile; +using ArrowROFile = arrow::io::ReadableFileInterface; namespace parquet { namespace arrow { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/arrow/io.h ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/io.h b/src/parquet/arrow/io.h index dc60635..119f8de 100644 --- a/src/parquet/arrow/io.h +++ b/src/parquet/arrow/io.h @@ -60,7 +60,7 @@ class PARQUET_EXPORT ParquetReadSource : public RandomAccessSource { explicit ParquetReadSource(ParquetAllocator* allocator); // We need to ask for the file size on opening the file, and this can fail - ::arrow::Status Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file); + ::arrow::Status Open(const std::shared_ptr<::arrow::io::ReadableFileInterface>& file); void Close() override; int64_t Tell() const override; @@ -70,7 +70,7 @@ class PARQUET_EXPORT ParquetReadSource : public RandomAccessSource { private: // An Arrow readable file of some kind - std::shared_ptr<::arrow::io::RandomAccessFile> file_; + std::shared_ptr<::arrow::io::ReadableFileInterface> file_; // The allocator is required for creating managed buffers ParquetAllocator* allocator_; http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/arrow/reader.cc ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/reader.cc b/src/parquet/arrow/reader.cc index 056b5ab..0e9f255 100644 --- a/src/parquet/arrow/reader.cc +++ b/src/parquet/arrow/reader.cc @@ -78,8 +78,8 @@ class FileReader::Impl { class FlatColumnReader::Impl { public: - Impl(MemoryPool* pool, const ColumnDescriptor* descr, - ParquetFileReader* reader, int column_index); + Impl(MemoryPool* pool, const ColumnDescriptor* descr, ParquetFileReader* reader, + int column_index); virtual ~Impl() {} Status NextBatch(int batch_size, std::shared_ptr<Array>* out); @@ -139,8 +139,7 @@ class FlatColumnReader::Impl { PoolBuffer valid_bytes_buffer_; }; -FileReader::Impl::Impl( - MemoryPool* pool, std::unique_ptr<ParquetFileReader> reader) +FileReader::Impl::Impl(MemoryPool* pool, std::unique_ptr<ParquetFileReader> reader) : pool_(pool), reader_(std::move(reader)) {} bool FileReader::Impl::CheckForFlatColumn(const ColumnDescriptor* descr) { @@ -191,14 +190,13 @@ Status FileReader::Impl::ReadFlatTable(std::shared_ptr<Table>* table) { return Status::OK(); } -FileReader::FileReader( - MemoryPool* pool, std::unique_ptr<ParquetFileReader> reader) +FileReader::FileReader(MemoryPool* pool, std::unique_ptr<ParquetFileReader> reader) : impl_(new FileReader::Impl(pool, std::move(reader))) {} FileReader::~FileReader() {} // Static ctor -Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file, +Status OpenFile(const std::shared_ptr<::arrow::io::ReadableFileInterface>& file, ParquetAllocator* allocator, std::unique_ptr<FileReader>* reader) { std::unique_ptr<ParquetReadSource> source(new ParquetReadSource(allocator)); RETURN_NOT_OK(source->Open(file)); @@ -320,8 +318,7 @@ Status FlatColumnReader::Impl::TypedReadBatch<::arrow::StringType, ByteArrayType if (descr_->max_definition_level() > 0) { def_levels_buffer_.Resize(values_to_read * sizeof(int16_t)); } - auto reader = - dynamic_cast<TypedColumnReader<ByteArrayType>*>(column_reader_.get()); + auto reader = dynamic_cast<TypedColumnReader<ByteArrayType>*>(column_reader_.get()); int64_t values_read; int64_t levels_read; int16_t* def_levels = reinterpret_cast<int16_t*>(def_levels_buffer_.mutable_data()); http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/arrow/reader.h ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/reader.h b/src/parquet/arrow/reader.h index e725728..7028be4 100644 --- a/src/parquet/arrow/reader.h +++ b/src/parquet/arrow/reader.h @@ -139,7 +139,7 @@ class PARQUET_EXPORT FlatColumnReader { // Helper function to create a file reader from an implementation of an Arrow // readable file PARQUET_EXPORT -::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file, +::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::ReadableFileInterface>& file, ParquetAllocator* allocator, std::unique_ptr<FileReader>* reader); } // namespace arrow http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/arrow/utils.h ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/utils.h b/src/parquet/arrow/utils.h index b443c99..bd53792 100644 --- a/src/parquet/arrow/utils.h +++ b/src/parquet/arrow/utils.h @@ -38,14 +38,14 @@ namespace arrow { (s); \ } catch (const ::parquet::ParquetException& e) {} -#define PARQUET_THROW_NOT_OK(s) \ - do { \ - ::arrow::Status _s = (s); \ - if (!_s.ok()) { \ - std::stringstream ss; \ - ss << "Arrow error: " << _s.ToString(); \ - throw ::parquet::ParquetException(ss.str()); \ - } \ +#define PARQUET_THROW_NOT_OK(s) \ + do { \ + ::arrow::Status _s = (s); \ + if (!_s.ok()) { \ + std::stringstream ss; \ + ss << "Arrow error: " << _s.ToString(); \ + ParquetException::Throw(ss.str()); \ + } \ } while (0); } // namespace arrow http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/arrow/writer.cc ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/writer.cc b/src/parquet/arrow/writer.cc index 5b5f41f..ff3707b 100644 --- a/src/parquet/arrow/writer.cc +++ b/src/parquet/arrow/writer.cc @@ -51,8 +51,8 @@ class FileWriter::Impl { Status NewRowGroup(int64_t chunk_size); template <typename ParquetType, typename ArrowType> - Status TypedWriteBatch(ColumnWriter* writer, const PrimitiveArray* data, - int64_t offset, int64_t length); + Status TypedWriteBatch( + ColumnWriter* writer, const PrimitiveArray* data, int64_t offset, int64_t length); // TODO(uwe): Same code as in reader.cc the only difference is the name of the temporary // buffer @@ -100,8 +100,7 @@ class FileWriter::Impl { RowGroupWriter* row_group_writer_; }; -FileWriter::Impl::Impl( - MemoryPool* pool, std::unique_ptr<ParquetFileWriter> writer) +FileWriter::Impl::Impl(MemoryPool* pool, std::unique_ptr<ParquetFileWriter> writer) : pool_(pool), data_buffer_(pool), writer_(std::move(writer)), @@ -121,8 +120,7 @@ Status FileWriter::Impl::TypedWriteBatch(ColumnWriter* column_writer, DCHECK((offset + length) <= data->length()); auto data_ptr = reinterpret_cast<const ArrowCType*>(data->data()->data()) + offset; - auto writer = - reinterpret_cast<TypedColumnWriter<ParquetType>*>(column_writer); + auto writer = reinterpret_cast<TypedColumnWriter<ParquetType>*>(column_writer); if (writer->descr()->max_definition_level() == 0) { // no nulls, just dump the data const ParquetCType* data_writer_ptr = nullptr; @@ -174,8 +172,7 @@ Status FileWriter::Impl::TypedWriteBatch<BooleanType, ::arrow::BooleanType>( RETURN_NOT_OK(data_buffer_.Resize(length)); auto data_ptr = reinterpret_cast<const uint8_t*>(data->data()->data()); auto buffer_ptr = reinterpret_cast<bool*>(data_buffer_.mutable_data()); - auto writer = reinterpret_cast<TypedColumnWriter<BooleanType>*>( - column_writer); + auto writer = reinterpret_cast<TypedColumnWriter<BooleanType>*>(column_writer); if (writer->descr()->max_definition_level() == 0) { // no nulls, just dump the data for (int64_t i = 0; i < length; i++) { @@ -266,8 +263,7 @@ Status FileWriter::Impl::WriteFlatColumnChunk( auto values = std::dynamic_pointer_cast<PrimitiveArray>(data->values()); auto data_ptr = reinterpret_cast<const uint8_t*>(values->data()->data()); DCHECK(values != nullptr); - auto writer = reinterpret_cast<TypedColumnWriter<ByteArrayType>*>( - column_writer); + auto writer = reinterpret_cast<TypedColumnWriter<ByteArrayType>*>(column_writer); if (writer->descr()->max_definition_level() > 0) { RETURN_NOT_OK(def_levels_buffer_.Resize(length * sizeof(int16_t))); } @@ -275,8 +271,8 @@ Status FileWriter::Impl::WriteFlatColumnChunk( if (writer->descr()->max_definition_level() == 0 || data->null_count() == 0) { // no nulls, just dump the data for (int64_t i = 0; i < length; i++) { - buffer_ptr[i] = ByteArray( - data->value_length(i + offset), data_ptr + data->value_offset(i)); + buffer_ptr[i] = + ByteArray(data->value_length(i + offset), data_ptr + data->value_offset(i)); } if (writer->descr()->max_definition_level() > 0) { std::fill(def_levels_ptr, def_levels_ptr + length, 1); @@ -301,8 +297,7 @@ Status FileWriter::Impl::WriteFlatColumnChunk( return Status::OK(); } -FileWriter::FileWriter( - MemoryPool* pool, std::unique_ptr<ParquetFileWriter> writer) +FileWriter::FileWriter(MemoryPool* pool, std::unique_ptr<ParquetFileWriter> writer) : impl_(new FileWriter::Impl(pool, std::move(writer))) {} Status FileWriter::NewRowGroup(int64_t chunk_size) { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/column/column-io-benchmark.cc ---------------------------------------------------------------------- diff --git a/src/parquet/column/column-io-benchmark.cc b/src/parquet/column/column-io-benchmark.cc index 319c8f5..bc4afec 100644 --- a/src/parquet/column/column-io-benchmark.cc +++ b/src/parquet/column/column-io-benchmark.cc @@ -17,10 +17,10 @@ #include "benchmark/benchmark.h" -#include "parquet/file/reader-internal.h" -#include "parquet/file/writer-internal.h" #include "parquet/column/reader.h" #include "parquet/column/writer.h" +#include "parquet/file/reader-internal.h" +#include "parquet/file/writer-internal.h" #include "parquet/util/input.h" namespace parquet { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/column/column-reader-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/column/column-reader-test.cc b/src/parquet/column/column-reader-test.cc index 524ec50..ee29f4b 100644 --- a/src/parquet/column/column-reader-test.cc +++ b/src/parquet/column/column-reader-test.cc @@ -25,12 +25,12 @@ #include <string> #include <vector> -#include "parquet/types.h" #include "parquet/column/page.h" #include "parquet/column/reader.h" #include "parquet/column/test-util.h" #include "parquet/schema/descriptor.h" #include "parquet/schema/types.h" +#include "parquet/types.h" #include "parquet/util/test-common.h" using std::string; http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/column/column-writer-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/column/column-writer-test.cc b/src/parquet/column/column-writer-test.cc index 57f1d1b..c843a20 100644 --- a/src/parquet/column/column-writer-test.cc +++ b/src/parquet/column/column-writer-test.cc @@ -17,16 +17,16 @@ #include <gtest/gtest.h> -#include "parquet/column/test-util.h" #include "parquet/column/test-specialization.h" +#include "parquet/column/test-util.h" -#include "parquet/file/reader-internal.h" -#include "parquet/file/writer-internal.h" #include "parquet/column/reader.h" #include "parquet/column/writer.h" +#include "parquet/file/reader-internal.h" +#include "parquet/file/writer-internal.h" +#include "parquet/types.h" #include "parquet/util/input.h" #include "parquet/util/output.h" -#include "parquet/types.h" namespace parquet { @@ -53,7 +53,9 @@ class TestPrimitiveWriter : public PrimitiveTypedTest<TestType> { definition_levels_out_.resize(SMALL_SIZE); repetition_levels_out_.resize(SMALL_SIZE); - this->SetUpSchemaRequired(); + this->SetUpSchema(Repetition::REQUIRED); + + descr_ = this->schema_.Column(0); } Type::type type_num() { return TestType::type_num; } @@ -63,15 +65,14 @@ class TestPrimitiveWriter : public PrimitiveTypedTest<TestType> { std::unique_ptr<InMemoryInputStream> source(new InMemoryInputStream(buffer)); std::unique_ptr<SerializedPageReader> page_reader( new SerializedPageReader(std::move(source), Compression::UNCOMPRESSED)); - reader_.reset( - new TypedColumnReader<TestType>(this->descr_.get(), std::move(page_reader))); + reader_.reset(new TypedColumnReader<TestType>(this->descr_, std::move(page_reader))); } std::shared_ptr<TypedColumnWriter<TestType>> BuildWriter( int64_t output_size = SMALL_SIZE, Encoding::type encoding = Encoding::PLAIN) { sink_.reset(new InMemoryOutputStream()); - metadata_ = ColumnChunkMetaDataBuilder::Make(writer_properties_, this->descr_.get(), - reinterpret_cast<uint8_t*>(&thrift_metadata_)); + metadata_ = ColumnChunkMetaDataBuilder::Make( + writer_properties_, this->descr_, reinterpret_cast<uint8_t*>(&thrift_metadata_)); std::unique_ptr<SerializedPageWriter> pager(new SerializedPageWriter( sink_.get(), Compression::UNCOMPRESSED, metadata_.get())); WriterProperties::Builder wp_builder; @@ -83,7 +84,7 @@ class TestPrimitiveWriter : public PrimitiveTypedTest<TestType> { } writer_properties_ = wp_builder.build(); std::shared_ptr<ColumnWriter> writer = ColumnWriter::Make( - this->descr_.get(), std::move(pager), output_size, writer_properties_.get()); + this->descr_, std::move(pager), output_size, writer_properties_.get()); return std::static_pointer_cast<TypedColumnWriter<TestType>>(writer); } @@ -137,16 +138,18 @@ class TestPrimitiveWriter : public PrimitiveTypedTest<TestType> { std::vector<int16_t> definition_levels_out_; std::vector<int16_t> repetition_levels_out_; + const ColumnDescriptor* descr_; + private: format::ColumnChunk thrift_metadata_; std::unique_ptr<ColumnChunkMetaDataBuilder> metadata_; - std::shared_ptr<ColumnDescriptor> schema_; std::unique_ptr<InMemoryOutputStream> sink_; std::shared_ptr<WriterProperties> writer_properties_; }; typedef ::testing::Types<Int32Type, Int64Type, Int96Type, FloatType, DoubleType, - BooleanType, ByteArrayType, FLBAType> TestTypes; + BooleanType, ByteArrayType, FLBAType> + TestTypes; TYPED_TEST_CASE(TestPrimitiveWriter, TestTypes); @@ -189,7 +192,7 @@ TYPED_TEST(TestPrimitiveWriter, RequiredRLEDictionary) { TYPED_TEST(TestPrimitiveWriter, Optional) { // Optional and non-repeated, with definition levels // but no repetition levels - this->SetUpSchemaOptional(); + this->SetUpSchema(Repetition::OPTIONAL); this->GenerateData(SMALL_SIZE); std::vector<int16_t> definition_levels(SMALL_SIZE, 1); @@ -212,7 +215,7 @@ TYPED_TEST(TestPrimitiveWriter, Optional) { TYPED_TEST(TestPrimitiveWriter, Repeated) { // Optional and repeated, so definition and repetition levels - this->SetUpSchemaRepeated(); + this->SetUpSchema(Repetition::REPEATED); this->GenerateData(SMALL_SIZE); std::vector<int16_t> definition_levels(SMALL_SIZE, 1); @@ -250,7 +253,7 @@ TYPED_TEST(TestPrimitiveWriter, RequiredTooMany) { TYPED_TEST(TestPrimitiveWriter, RepeatedTooFewRows) { // Optional and repeated, so definition and repetition levels - this->SetUpSchemaRepeated(); + this->SetUpSchema(Repetition::REPEATED); this->GenerateData(SMALL_SIZE); std::vector<int16_t> definition_levels(SMALL_SIZE, 1); @@ -308,7 +311,7 @@ TYPED_TEST(TestPrimitiveWriter, RequiredVeryLargeChunk) { // PARQUET-719 // Test case for NULL values TEST_F(TestNullValuesWriter, OptionalNullValueChunk) { - this->SetUpSchemaOptional(); + this->SetUpSchema(Repetition::OPTIONAL); this->GenerateData(LARGE_SIZE); http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/column/levels-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/column/levels-test.cc b/src/parquet/column/levels-test.cc index 126873c..cee2763 100644 --- a/src/parquet/column/levels-test.cc +++ b/src/parquet/column/levels-test.cc @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include <gtest/gtest.h> #include <cstdint> +#include <gtest/gtest.h> #include <memory> -#include <vector> #include <string> +#include <vector> #include "parquet/column/levels.h" #include "parquet/types.h" http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/column/levels.h ---------------------------------------------------------------------- diff --git a/src/parquet/column/levels.h b/src/parquet/column/levels.h index c57ca2f..f676db2 100644 --- a/src/parquet/column/levels.h +++ b/src/parquet/column/levels.h @@ -18,8 +18,8 @@ #ifndef PARQUET_COLUMN_LEVELS_H #define PARQUET_COLUMN_LEVELS_H -#include <memory> #include <algorithm> +#include <memory> #include "parquet/exception.h" #include "parquet/types.h" http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/column/properties.h ---------------------------------------------------------------------- diff --git a/src/parquet/column/properties.h b/src/parquet/column/properties.h index 91a4672..cdd1a12 100644 --- a/src/parquet/column/properties.h +++ b/src/parquet/column/properties.h @@ -23,8 +23,8 @@ #include <unordered_map> #include "parquet/exception.h" -#include "parquet/types.h" #include "parquet/schema/types.h" +#include "parquet/types.h" #include "parquet/util/input.h" #include "parquet/util/mem-allocator.h" #include "parquet/util/visibility.h" http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/column/scanner-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/column/scanner-test.cc b/src/parquet/column/scanner-test.cc index fb5178a..8eee191 100644 --- a/src/parquet/column/scanner-test.cc +++ b/src/parquet/column/scanner-test.cc @@ -24,14 +24,14 @@ #include <string> #include <vector> -#include "parquet/types.h" #include "parquet/column/page.h" #include "parquet/column/scanner.h" +#include "parquet/column/test-specialization.h" #include "parquet/column/test-util.h" #include "parquet/schema/descriptor.h" #include "parquet/schema/types.h" +#include "parquet/types.h" #include "parquet/util/test-common.h" -#include "parquet/column/test-specialization.h" using std::string; using std::vector; @@ -146,7 +146,8 @@ static int num_pages = 20; static int batch_size = 32; typedef ::testing::Types<Int32Type, Int64Type, Int96Type, FloatType, DoubleType, - ByteArrayType> TestTypes; + ByteArrayType> + TestTypes; using TestBooleanFlatScanner = TestFlatScanner<BooleanType>; using TestFLBAFlatScanner = TestFlatScanner<FLBAType>; http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/column/scanner.h ---------------------------------------------------------------------- diff --git a/src/parquet/column/scanner.h b/src/parquet/column/scanner.h index 4373c7a..184c74d 100644 --- a/src/parquet/column/scanner.h +++ b/src/parquet/column/scanner.h @@ -18,10 +18,10 @@ #ifndef PARQUET_COLUMN_SCANNER_H #define PARQUET_COLUMN_SCANNER_H -#include <stdio.h> #include <cstdint> #include <memory> #include <ostream> +#include <stdio.h> #include <string> #include <vector> http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/column/test-specialization.h ---------------------------------------------------------------------- diff --git a/src/parquet/column/test-specialization.h b/src/parquet/column/test-specialization.h index 5803b65..27781cc 100644 --- a/src/parquet/column/test-specialization.h +++ b/src/parquet/column/test-specialization.h @@ -24,6 +24,8 @@ #include <algorithm> #include <limits> +#include <sstream> +#include <string> #include <vector> #include "parquet/column/test-util.h" @@ -60,52 +62,43 @@ void InitValues<Int96>(int num_values, vector<Int96>& values, vector<uint8_t>& b std::numeric_limits<int32_t>::max(), values.data()); } +inline std::string TestColumnName(int i) { + std::stringstream col_name; + col_name << "column_" << i; + return col_name.str(); +} + // This class lives here because of its dependency on the InitValues specializations. template <typename TestType> class PrimitiveTypedTest : public ::testing::Test { public: typedef typename TestType::c_type T; - void SetUpSchemaRequired() { - primitive_node_ = schema::PrimitiveNode::Make("column", Repetition::REQUIRED, - TestType::type_num, LogicalType::NONE, FLBA_LENGTH); - descr_ = std::make_shared<ColumnDescriptor>(primitive_node_, 0, 0); - node_ = schema::GroupNode::Make( - "schema", Repetition::REQUIRED, std::vector<schema::NodePtr>({primitive_node_})); - schema_.Init(node_); - } - - void SetUpSchemaOptional() { - primitive_node_ = schema::PrimitiveNode::Make("column", Repetition::OPTIONAL, - TestType::type_num, LogicalType::NONE, FLBA_LENGTH); - descr_ = std::make_shared<ColumnDescriptor>(primitive_node_, 1, 0); - node_ = schema::GroupNode::Make( - "schema", Repetition::REQUIRED, std::vector<schema::NodePtr>({primitive_node_})); - schema_.Init(node_); - } + void SetUpSchema(Repetition::type repetition, int num_columns = 1) { + std::vector<schema::NodePtr> fields; - void SetUpSchemaRepeated() { - primitive_node_ = schema::PrimitiveNode::Make("column", Repetition::REPEATED, - TestType::type_num, LogicalType::NONE, FLBA_LENGTH); - descr_ = std::make_shared<ColumnDescriptor>(primitive_node_, 1, 1); - node_ = schema::GroupNode::Make( - "schema", Repetition::REQUIRED, std::vector<schema::NodePtr>({primitive_node_})); + for (int i = 0; i < num_columns; ++i) { + std::string name = TestColumnName(i); + fields.push_back(schema::PrimitiveNode::Make( + name, repetition, TestType::type_num, LogicalType::NONE, FLBA_LENGTH)); + } + node_ = schema::GroupNode::Make("schema", Repetition::REQUIRED, fields); schema_.Init(node_); } void GenerateData(int64_t num_values); void SetupValuesOut(int64_t num_values); void SyncValuesOut(); - void SetUp() { SetUpSchemaRequired(); } protected: - schema::NodePtr primitive_node_; schema::NodePtr node_; SchemaDescriptor schema_; - std::shared_ptr<ColumnDescriptor> descr_; // Input buffers std::vector<T> values_; + + std::vector<int16_t> def_levels_; + std::vector<uint8_t> buffer_; // Pointer to the values, needed as we cannot use vector<bool>::data() T* values_ptr_; @@ -127,13 +120,17 @@ void PrimitiveTypedTest<BooleanType>::SyncValuesOut() { template <typename TestType> void PrimitiveTypedTest<TestType>::SetupValuesOut(int64_t num_values) { + values_out_.clear(); values_out_.resize(num_values); values_out_ptr_ = values_out_.data(); } template <> void PrimitiveTypedTest<BooleanType>::SetupValuesOut(int64_t num_values) { + values_out_.clear(); values_out_.resize(num_values); + + bool_buffer_out_.clear(); bool_buffer_out_.resize(num_values); // Write once to all values so we can copy it without getting Valgrind errors // about uninitialised values. @@ -143,18 +140,26 @@ void PrimitiveTypedTest<BooleanType>::SetupValuesOut(int64_t num_values) { template <typename TestType> void PrimitiveTypedTest<TestType>::GenerateData(int64_t num_values) { + def_levels_.resize(num_values); values_.resize(num_values); + InitValues<T>(num_values, values_, buffer_); values_ptr_ = values_.data(); + + std::fill(def_levels_.begin(), def_levels_.end(), 1); } template <> void PrimitiveTypedTest<BooleanType>::GenerateData(int64_t num_values) { + def_levels_.resize(num_values); values_.resize(num_values); + InitValues<T>(num_values, values_, buffer_); bool_buffer_.resize(num_values); std::copy(values_.begin(), values_.end(), bool_buffer_.begin()); values_ptr_ = reinterpret_cast<bool*>(bool_buffer_.data()); + + std::fill(def_levels_.begin(), def_levels_.end(), 1); } } // namespace test http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/column/test-util.h ---------------------------------------------------------------------- diff --git a/src/parquet/column/test-util.h b/src/parquet/column/test-util.h index b11e620..10632d2 100644 --- a/src/parquet/column/test-util.h +++ b/src/parquet/column/test-util.h @@ -25,15 +25,15 @@ #include <algorithm> #include <limits> #include <memory> -#include <vector> #include <string> +#include <vector> #include "parquet/column/levels.h" #include "parquet/column/page.h" // Depended on by SerializedPageReader test utilities for now -#include "parquet/encodings/plain-encoding.h" #include "parquet/encodings/dictionary-encoding.h" +#include "parquet/encodings/plain-encoding.h" #include "parquet/util/input.h" #include "parquet/util/test-common.h" http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/compression/codec-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/compression/codec-test.cc b/src/parquet/compression/codec-test.cc index 417ecc7..2f7cc1a 100644 --- a/src/parquet/compression/codec-test.cc +++ b/src/parquet/compression/codec-test.cc @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include <gtest/gtest.h> #include <cstdint> +#include <gtest/gtest.h> #include <string> #include <vector> http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/compression/snappy-codec.cc ---------------------------------------------------------------------- diff --git a/src/parquet/compression/snappy-codec.cc b/src/parquet/compression/snappy-codec.cc index ccd25b9..70f3a41 100644 --- a/src/parquet/compression/snappy-codec.cc +++ b/src/parquet/compression/snappy-codec.cc @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include <snappy.h> #include <cstdint> #include <cstdlib> +#include <snappy.h> #include "parquet/compression/codec.h" #include "parquet/exception.h" http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/encodings/delta-byte-array-encoding.h ---------------------------------------------------------------------- diff --git a/src/parquet/encodings/delta-byte-array-encoding.h b/src/parquet/encodings/delta-byte-array-encoding.h index d208f26..827789d 100644 --- a/src/parquet/encodings/delta-byte-array-encoding.h +++ b/src/parquet/encodings/delta-byte-array-encoding.h @@ -21,8 +21,8 @@ #include <algorithm> #include "parquet/encodings/decoder.h" -#include "parquet/encodings/delta-length-byte-array-encoding.h" #include "parquet/encodings/delta-bit-pack-encoding.h" +#include "parquet/encodings/delta-length-byte-array-encoding.h" namespace parquet { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/encodings/encoding-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/encodings/encoding-test.cc b/src/parquet/encodings/encoding-test.cc index 50adc9e..daa25cb 100644 --- a/src/parquet/encodings/encoding-test.cc +++ b/src/parquet/encodings/encoding-test.cc @@ -15,18 +15,18 @@ // specific language governing permissions and limitations // under the License. -#include <gtest/gtest.h> #include <cstdint> #include <cstdlib> #include <cstring> +#include <gtest/gtest.h> #include <string> #include <vector> -#include "parquet/schema/descriptor.h" #include "parquet/encodings/dictionary-encoding.h" #include "parquet/encodings/plain-encoding.h" -#include "parquet/types.h" +#include "parquet/schema/descriptor.h" #include "parquet/schema/types.h" +#include "parquet/types.h" #include "parquet/util/bit-util.h" #include "parquet/util/buffer.h" #include "parquet/util/output.h" @@ -238,7 +238,8 @@ TYPED_TEST(TestPlainEncoding, BasicRoundTrip) { // Dictionary encoding tests typedef ::testing::Types<Int32Type, Int64Type, Int96Type, FloatType, DoubleType, - ByteArrayType, FLBAType> DictEncodedTypes; + ByteArrayType, FLBAType> + DictEncodedTypes; template <typename Type> class TestDictionaryEncoding : public TestEncodingBase<Type> { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/exception.cc ---------------------------------------------------------------------- diff --git a/src/parquet/exception.cc b/src/parquet/exception.cc index 0973e5a..96bbc4b 100644 --- a/src/parquet/exception.cc +++ b/src/parquet/exception.cc @@ -33,6 +33,10 @@ void ParquetException::NYI(const std::string& msg) { throw ParquetException(ss.str()); } +void ParquetException::Throw(const std::string& msg) { + throw ParquetException(msg); +} + ParquetException::ParquetException(const char* msg) : msg_(msg) {} ParquetException::ParquetException(const std::string& msg) : msg_(msg) {} http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/exception.h ---------------------------------------------------------------------- diff --git a/src/parquet/exception.h b/src/parquet/exception.h index 3851018..b161bf7 100644 --- a/src/parquet/exception.h +++ b/src/parquet/exception.h @@ -29,6 +29,7 @@ class PARQUET_EXPORT ParquetException : public std::exception { public: static void EofException(); static void NYI(const std::string& msg); + static void Throw(const std::string& msg); explicit ParquetException(const char* msg); explicit ParquetException(const std::string& msg); http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/file/file-deserialize-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/file/file-deserialize-test.cc b/src/parquet/file/file-deserialize-test.cc index db99f16..adc90ed 100644 --- a/src/parquet/file/file-deserialize-test.cc +++ b/src/parquet/file/file-deserialize-test.cc @@ -18,8 +18,8 @@ #include <gtest/gtest.h> #include <algorithm> -#include <cstdlib> #include <cstdint> +#include <cstdlib> #include <cstring> #include <exception> #include <memory> http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/file/file-metadata-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/file/file-metadata-test.cc b/src/parquet/file/file-metadata-test.cc index a7f83c5..576db43 100644 --- a/src/parquet/file/file-metadata-test.cc +++ b/src/parquet/file/file-metadata-test.cc @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include <gtest/gtest.h> #include "parquet/file/metadata.h" #include "parquet/schema/descriptor.h" #include "parquet/schema/types.h" #include "parquet/types.h" +#include <gtest/gtest.h> namespace parquet { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/file/file-serialize-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/file/file-serialize-test.cc b/src/parquet/file/file-serialize-test.cc index 90ee7de..42a73c9 100644 --- a/src/parquet/file/file-serialize-test.cc +++ b/src/parquet/file/file-serialize-test.cc @@ -18,8 +18,8 @@ #include <gtest/gtest.h> #include "parquet/column/reader.h" -#include "parquet/column/test-util.h" #include "parquet/column/test-specialization.h" +#include "parquet/column/test-util.h" #include "parquet/column/writer.h" #include "parquet/file/reader.h" #include "parquet/file/writer.h" @@ -40,51 +40,74 @@ class TestSerialize : public PrimitiveTypedTest<TestType> { public: typedef typename TestType::c_type T; + void SetUp() { + num_columns_ = 4; + this->SetUpSchema(Repetition::OPTIONAL, num_columns_); + } + protected: + int num_columns_; + void FileSerializeTest(Compression::type codec_type) { std::shared_ptr<InMemoryOutputStream> sink(new InMemoryOutputStream()); auto gnode = std::static_pointer_cast<GroupNode>(this->node_); - std::shared_ptr<WriterProperties> writer_properties = - WriterProperties::Builder().compression("column", codec_type)->build(); + + WriterProperties::Builder prop_builder; + + for (int i = 0; i < num_columns_; ++i) { + prop_builder.compression(this->schema_.Column(i)->name(), codec_type); + } + std::shared_ptr<WriterProperties> writer_properties = prop_builder.build(); + auto file_writer = ParquetFileWriter::Open(sink, gnode, writer_properties); auto row_group_writer = file_writer->AppendRowGroup(100); - auto column_writer = - static_cast<TypedColumnWriter<TestType>*>(row_group_writer->NextColumn()); + this->GenerateData(100); - column_writer->WriteBatch(100, nullptr, nullptr, this->values_ptr_); - column_writer->Close(); + for (int i = 0; i < num_columns_; ++i) { + auto column_writer = + static_cast<TypedColumnWriter<TestType>*>(row_group_writer->NextColumn()); + column_writer->WriteBatch( + 100, this->def_levels_.data(), nullptr, this->values_ptr_); + column_writer->Close(); + } + row_group_writer->Close(); file_writer->Close(); auto buffer = sink->GetBuffer(); std::unique_ptr<RandomAccessSource> source(new BufferReader(buffer)); auto file_reader = ParquetFileReader::Open(std::move(source)); - ASSERT_EQ(1, file_reader->metadata()->num_columns()); + ASSERT_EQ(num_columns_, file_reader->metadata()->num_columns()); ASSERT_EQ(1, file_reader->metadata()->num_row_groups()); ASSERT_EQ(100, file_reader->metadata()->num_rows()); auto rg_reader = file_reader->RowGroup(0); - ASSERT_EQ(1, rg_reader->metadata()->num_columns()); + ASSERT_EQ(num_columns_, rg_reader->metadata()->num_columns()); ASSERT_EQ(100, rg_reader->metadata()->num_rows()); // Check that the specified compression was actually used. ASSERT_EQ(codec_type, rg_reader->metadata()->ColumnChunk(0)->compression()); - auto col_reader = - std::static_pointer_cast<TypedColumnReader<TestType>>(rg_reader->Column(0)); - std::vector<int16_t> def_levels_out(100); - std::vector<int16_t> rep_levels_out(100); int64_t values_read; - this->SetupValuesOut(100); - col_reader->ReadBatch(100, def_levels_out.data(), rep_levels_out.data(), - this->values_out_ptr_, &values_read); - this->SyncValuesOut(); - ASSERT_EQ(100, values_read); - ASSERT_EQ(this->values_, this->values_out_); + + for (int i = 0; i < num_columns_; ++i) { + std::vector<int16_t> def_levels_out(100); + std::vector<int16_t> rep_levels_out(100); + auto col_reader = + std::static_pointer_cast<TypedColumnReader<TestType>>(rg_reader->Column(i)); + this->SetupValuesOut(100); + col_reader->ReadBatch(100, def_levels_out.data(), rep_levels_out.data(), + this->values_out_ptr_, &values_read); + this->SyncValuesOut(); + ASSERT_EQ(100, values_read); + ASSERT_EQ(this->values_, this->values_out_); + ASSERT_EQ(this->def_levels_, def_levels_out); + } } }; typedef ::testing::Types<Int32Type, Int64Type, Int96Type, FloatType, DoubleType, - BooleanType, ByteArrayType, FLBAType> TestTypes; + BooleanType, ByteArrayType, FLBAType> + TestTypes; TYPED_TEST_CASE(TestSerialize, TestTypes); http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/file/metadata.h ---------------------------------------------------------------------- diff --git a/src/parquet/file/metadata.h b/src/parquet/file/metadata.h index 0ef6fa0..b36ced8 100644 --- a/src/parquet/file/metadata.h +++ b/src/parquet/file/metadata.h @@ -18,9 +18,9 @@ #ifndef PARQUET_FILE_METADATA_H #define PARQUET_FILE_METADATA_H +#include <set> #include <string> #include <vector> -#include <set> #include "parquet/column/properties.h" #include "parquet/compression/codec.h" http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/file/reader-internal.cc ---------------------------------------------------------------------- diff --git a/src/parquet/file/reader-internal.cc b/src/parquet/file/reader-internal.cc index 1cb91f0..8477052 100644 --- a/src/parquet/file/reader-internal.cc +++ b/src/parquet/file/reader-internal.cc @@ -17,10 +17,10 @@ #include "parquet/file/reader-internal.h" -#include <string.h> #include <algorithm> #include <exception> #include <ostream> +#include <string.h> #include <string> #include <vector> @@ -196,8 +196,8 @@ SerializedFile::~SerializedFile() { } std::shared_ptr<RowGroupReader> SerializedFile::GetRowGroup(int i) { - std::unique_ptr<SerializedRowGroup> contents(new SerializedRowGroup( - source_.get(), std::move(file_metadata_->RowGroup(i)), properties_)); + std::unique_ptr<SerializedRowGroup> contents( + new SerializedRowGroup(source_.get(), file_metadata_->RowGroup(i), properties_)); return std::make_shared<RowGroupReader>(std::move(contents)); } http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/file/reader.cc ---------------------------------------------------------------------- diff --git a/src/parquet/file/reader.cc b/src/parquet/file/reader.cc index 7cf3f1a..3672d94 100644 --- a/src/parquet/file/reader.cc +++ b/src/parquet/file/reader.cc @@ -29,9 +29,9 @@ #include "parquet/column/scanner.h" #include "parquet/exception.h" #include "parquet/file/reader-internal.h" +#include "parquet/types.h" #include "parquet/util/input.h" #include "parquet/util/logging.h" -#include "parquet/types.h" using std::string; using std::vector; http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/file/reader.h ---------------------------------------------------------------------- diff --git a/src/parquet/file/reader.h b/src/parquet/file/reader.h index c1ee9d4..bc92af3 100644 --- a/src/parquet/file/reader.h +++ b/src/parquet/file/reader.h @@ -20,8 +20,8 @@ #include <cstdint> #include <iosfwd> -#include <memory> #include <list> +#include <memory> #include <string> #include <vector> http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/file/writer-internal.cc ---------------------------------------------------------------------- diff --git a/src/parquet/file/writer-internal.cc b/src/parquet/file/writer-internal.cc index 05aefb9..554e779 100644 --- a/src/parquet/file/writer-internal.cc +++ b/src/parquet/file/writer-internal.cc @@ -142,7 +142,7 @@ ColumnWriter* RowGroupSerializer::NextColumn() { // Throws an error if more columns are being written auto col_meta = metadata_->NextColumnChunk(); - if (current_column_writer_) { total_bytes_written_ = current_column_writer_->Close(); } + if (current_column_writer_) { total_bytes_written_ += current_column_writer_->Close(); } const ColumnDescriptor* column_descr = col_meta->descr(); std::unique_ptr<PageWriter> pager( @@ -158,7 +158,7 @@ void RowGroupSerializer::Close() { closed_ = true; if (current_column_writer_) { - total_bytes_written_ = current_column_writer_->Close(); + total_bytes_written_ += current_column_writer_->Close(); current_column_writer_.reset(); } // Ensures all columns have been written http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/reader-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/reader-test.cc b/src/parquet/reader-test.cc index 2e28c80..04437b2 100644 --- a/src/parquet/reader-test.cc +++ b/src/parquet/reader-test.cc @@ -15,18 +15,18 @@ // specific language governing permissions and limitations // under the License. -#include <gtest/gtest.h> -#include <fcntl.h> -#include <cstdlib> #include <cstdint> +#include <cstdlib> +#include <fcntl.h> +#include <gtest/gtest.h> #include <iostream> #include <memory> #include <string> -#include "parquet/file/reader.h" -#include "parquet/file/reader-internal.h" #include "parquet/column/reader.h" #include "parquet/column/scanner.h" +#include "parquet/file/reader-internal.h" +#include "parquet/file/reader.h" #include "parquet/util/input.h" #include "parquet/util/mem-allocator.h" http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/schema/descriptor.h ---------------------------------------------------------------------- diff --git a/src/parquet/schema/descriptor.h b/src/parquet/schema/descriptor.h index c591954..1673d5f 100644 --- a/src/parquet/schema/descriptor.h +++ b/src/parquet/schema/descriptor.h @@ -18,15 +18,15 @@ #ifndef PARQUET_SCHEMA_DESCRIPTOR_H #define PARQUET_SCHEMA_DESCRIPTOR_H +#include "parquet/schema/types.h" +#include "parquet/types.h" +#include "parquet/util/visibility.h" #include <cstdint> #include <cstdlib> #include <memory> #include <string> #include <unordered_map> #include <vector> -#include "parquet/schema/types.h" -#include "parquet/types.h" -#include "parquet/util/visibility.h" namespace parquet { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/schema/schema-descriptor-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/schema/schema-descriptor-test.cc b/src/parquet/schema/schema-descriptor-test.cc index 5be8db7..eeaec5b 100644 --- a/src/parquet/schema/schema-descriptor-test.cc +++ b/src/parquet/schema/schema-descriptor-test.cc @@ -17,9 +17,9 @@ // Schema / column descriptor correctness tests (from flat Parquet schemas) -#include <gtest/gtest.h> #include <cstdint> #include <cstdlib> +#include <gtest/gtest.h> #include <string> #include <vector> http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/thrift/util.h ---------------------------------------------------------------------- diff --git a/src/parquet/thrift/util.h b/src/parquet/thrift/util.h index 38b1a15..a340c6c 100644 --- a/src/parquet/thrift/util.h +++ b/src/parquet/thrift/util.h @@ -9,13 +9,13 @@ // TCompactProtocol requires some #defines to work right. #define SIGNED_RIGHT_SHIFT_IS 1 #define ARITHMETIC_RIGHT_SHIFT 1 +#include <thrift/TApplicationException.h> #include <thrift/protocol/TCompactProtocol.h> #include <thrift/protocol/TDebugProtocol.h> -#include <thrift/TApplicationException.h> +#include <sstream> #include <thrift/protocol/TBinaryProtocol.h> #include <thrift/transport/TBufferTransports.h> -#include <sstream> #include "parquet/exception.h" #include "parquet/thrift/parquet_types.h" @@ -82,7 +82,8 @@ inline void DeserializeThriftMsg(const uint8_t* buf, uint32_t* len, T* deseriali boost::shared_ptr<apache::thrift::transport::TMemoryBuffer> tmem_transport( new apache::thrift::transport::TMemoryBuffer(const_cast<uint8_t*>(buf), *len)); apache::thrift::protocol::TCompactProtocolFactoryT< - apache::thrift::transport::TMemoryBuffer> tproto_factory; + apache::thrift::transport::TMemoryBuffer> + tproto_factory; boost::shared_ptr<apache::thrift::protocol::TProtocol> tproto = tproto_factory.getProtocol(tmem_transport); try { @@ -104,7 +105,8 @@ inline void SerializeThriftMsg(T* obj, uint32_t len, OutputStream* out) { boost::shared_ptr<apache::thrift::transport::TMemoryBuffer> mem_buffer( new apache::thrift::transport::TMemoryBuffer(len)); apache::thrift::protocol::TCompactProtocolFactoryT< - apache::thrift::transport::TMemoryBuffer> tproto_factory; + apache::thrift::transport::TMemoryBuffer> + tproto_factory; boost::shared_ptr<apache::thrift::protocol::TProtocol> tproto = tproto_factory.getProtocol(mem_buffer); try { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/util/bit-stream-utils.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/bit-stream-utils.h b/src/parquet/util/bit-stream-utils.h index f7d09a9..1fdf563 100644 --- a/src/parquet/util/bit-stream-utils.h +++ b/src/parquet/util/bit-stream-utils.h @@ -20,13 +20,13 @@ #ifndef PARQUET_UTIL_BIT_STREAM_UTILS_H #define PARQUET_UTIL_BIT_STREAM_UTILS_H -#include <string.h> #include <algorithm> #include <cstdint> +#include <string.h> +#include "parquet/util/bit-util.h" #include "parquet/util/compiler-util.h" #include "parquet/util/logging.h" -#include "parquet/util/bit-util.h" namespace parquet { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/util/bit-util-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/bit-util-test.cc b/src/parquet/util/bit-util-test.cc index f305b9c..bc3e182 100644 --- a/src/parquet/util/bit-util-test.cc +++ b/src/parquet/util/bit-util-test.cc @@ -17,17 +17,17 @@ // From Apache Impala as of 2016-01-29 -#include <stdlib.h> -#include <stdio.h> -#include <limits.h> #include <gtest/gtest.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> #include <boost/utility.hpp> #include <iostream> -#include "parquet/util/bit-util.h" #include "parquet/util/bit-stream-utils.inline.h" +#include "parquet/util/bit-util.h" #include "parquet/util/cpu-info.h" namespace parquet { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/util/buffer-builder.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/buffer-builder.h b/src/parquet/util/buffer-builder.h index 26f134e..b72e70d 100644 --- a/src/parquet/util/buffer-builder.h +++ b/src/parquet/util/buffer-builder.h @@ -20,8 +20,8 @@ #ifndef PARQUET_UTIL_BUFFER_BUILDER_H #define PARQUET_UTIL_BUFFER_BUILDER_H -#include <stdlib.h> #include <cstdint> +#include <stdlib.h> namespace parquet { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/util/buffer-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/buffer-test.cc b/src/parquet/util/buffer-test.cc index a71a1f6..ee5b000 100644 --- a/src/parquet/util/buffer-test.cc +++ b/src/parquet/util/buffer-test.cc @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -#include <gtest/gtest.h> -#include <cstdlib> #include <cstdint> +#include <cstdlib> #include <exception> +#include <gtest/gtest.h> #include <limits> #include <memory> #include <string> http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/util/cpu-info.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/cpu-info.cc b/src/parquet/util/cpu-info.cc index 1b1bc00..e8f6fac 100644 --- a/src/parquet/util/cpu-info.cc +++ b/src/parquet/util/cpu-info.cc @@ -32,8 +32,8 @@ #include <algorithm> #include <cstdint> -#include <iostream> #include <fstream> +#include <iostream> #include <sstream> #include <string> http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/util/input.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/input.cc b/src/parquet/util/input.cc index e1659f7..10e1573 100644 --- a/src/parquet/util/input.cc +++ b/src/parquet/util/input.cc @@ -17,10 +17,10 @@ #include "parquet/util/input.h" -#include <sys/mman.h> #include <algorithm> #include <sstream> #include <string> +#include <sys/mman.h> #include "parquet/exception.h" #include "parquet/util/buffer.h" http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/util/mem-pool-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/mem-pool-test.cc b/src/parquet/util/mem-pool-test.cc index 7e1443a..3f3424b 100644 --- a/src/parquet/util/mem-pool-test.cc +++ b/src/parquet/util/mem-pool-test.cc @@ -18,13 +18,13 @@ // Initially imported from Apache Impala on 2016-02-23, and has been modified // since for parquet-cpp -#include <gtest/gtest.h> #include <cstdint> +#include <gtest/gtest.h> #include <limits> #include <string> -#include "parquet/util/mem-pool.h" #include "parquet/util/bit-util.h" +#include "parquet/util/mem-pool.h" namespace parquet { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/util/mem-pool.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/mem-pool.h b/src/parquet/util/mem-pool.h index 496665d..5f6afa9 100644 --- a/src/parquet/util/mem-pool.h +++ b/src/parquet/util/mem-pool.h @@ -21,11 +21,11 @@ #ifndef PARQUET_UTIL_MEM_POOL_H #define PARQUET_UTIL_MEM_POOL_H -#include <stdio.h> #include <algorithm> #include <cstdint> -#include <vector> +#include <stdio.h> #include <string> +#include <vector> #include "parquet/util/mem-allocator.h" http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/util/rle-encoding.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/rle-encoding.h b/src/parquet/util/rle-encoding.h index 8fa1c41..c6b9577 100644 --- a/src/parquet/util/rle-encoding.h +++ b/src/parquet/util/rle-encoding.h @@ -20,13 +20,13 @@ #ifndef PARQUET_UTIL_RLE_ENCODING_H #define PARQUET_UTIL_RLE_ENCODING_H -#include <math.h> #include <algorithm> +#include <math.h> -#include "parquet/util/compiler-util.h" #include "parquet/util/bit-stream-utils.inline.h" #include "parquet/util/bit-util.h" #include "parquet/util/buffer.h" +#include "parquet/util/compiler-util.h" namespace parquet { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/util/rle-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/util/rle-test.cc b/src/parquet/util/rle-test.cc index c7d95d4..e50cd72 100644 --- a/src/parquet/util/rle-test.cc +++ b/src/parquet/util/rle-test.cc @@ -19,8 +19,8 @@ #include <gtest/gtest.h> #include <math.h> -#include <stdlib.h> #include <stdio.h> +#include <stdlib.h> #include <boost/utility.hpp> @@ -29,8 +29,8 @@ #include <random> #include <vector> -#include "parquet/util/rle-encoding.h" #include "parquet/util/bit-stream-utils.inline.h" +#include "parquet/util/rle-encoding.h" using std::vector; http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/util/stopwatch.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/stopwatch.h b/src/parquet/util/stopwatch.h index b940d8c..ad50267 100644 --- a/src/parquet/util/stopwatch.h +++ b/src/parquet/util/stopwatch.h @@ -21,8 +21,8 @@ #include <stdio.h> #include <sys/time.h> -#include <iostream> #include <ctime> +#include <iostream> namespace parquet { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/src/parquet/util/test-common.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/test-common.h b/src/parquet/util/test-common.h index edadb53..2327aeb 100644 --- a/src/parquet/util/test-common.h +++ b/src/parquet/util/test-common.h @@ -32,7 +32,8 @@ namespace parquet { namespace test { typedef ::testing::Types<BooleanType, Int32Type, Int64Type, Int96Type, FloatType, - DoubleType, ByteArrayType, FLBAType> ParquetTypes; + DoubleType, ByteArrayType, FLBAType> + ParquetTypes; template <typename T> static inline void assert_vector_equal(const vector<T>& left, const vector<T>& right) { http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/thirdparty/versions.sh ---------------------------------------------------------------------- diff --git a/thirdparty/versions.sh b/thirdparty/versions.sh index 05f5cc2..e60ce05 100755 --- a/thirdparty/versions.sh +++ b/thirdparty/versions.sh @@ -1,4 +1,4 @@ -ARROW_VERSION="6b8abb4402ff1f39fc5944a7df6e3b4755691d87" +ARROW_VERSION="7e39747eec05379710e1a42ecbaf1d9795bc3cf0" ARROW_URL="https://github.com/apache/arrow/archive/${ARROW_VERSION}.tar.gz"; ARROW_BASEDIR="arrow-${ARROW_VERSION}" http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/tools/parquet-scan.cc ---------------------------------------------------------------------- diff --git a/tools/parquet-scan.cc b/tools/parquet-scan.cc index d146a1d..f0bbb8e 100644 --- a/tools/parquet-scan.cc +++ b/tools/parquet-scan.cc @@ -17,8 +17,8 @@ #include <ctime> #include <iostream> -#include <memory> #include <list> +#include <memory> #include "parquet/api/reader.h" http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/5c1d9e94/tools/parquet_reader.cc ---------------------------------------------------------------------- diff --git a/tools/parquet_reader.cc b/tools/parquet_reader.cc index ced84d5..bc0711f 100644 --- a/tools/parquet_reader.cc +++ b/tools/parquet_reader.cc @@ -16,8 +16,8 @@ // under the License. #include <iostream> -#include <memory> #include <list> +#include <memory> #include "parquet/api/reader.h"
