Repository: parquet-cpp Updated Branches: refs/heads/master 6466cfd3a -> 005767c1c
PARQUET-1104: Upgrade to Apache Arrow 0.7.0 RC0 This will enable most of the outstanding PRs to be rebased and then merged Close #384 Close #394 Author: Wes McKinney <[email protected]> Closes #397 from wesm/PARQUET-1104 and squashes the following commits: 4106c5b [Wes McKinney] Include boost_system in static toolchain until ARROW-1536 resolved 005ee74 [Wes McKinney] Account for internal API change in ARROW-1511 3cab00d [Wes McKinney] Change MutableBuffer to Buffer to account for deprecation in ARROW-1402 7cd9702 [Wes McKinney] Upgrade to Apache Arrow 0.7.0 RC0 Project: http://git-wip-us.apache.org/repos/asf/parquet-cpp/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-cpp/commit/005767c1 Tree: http://git-wip-us.apache.org/repos/asf/parquet-cpp/tree/005767c1 Diff: http://git-wip-us.apache.org/repos/asf/parquet-cpp/diff/005767c1 Branch: refs/heads/master Commit: 005767c1ca69d95d62d5ae6cc47b06c6d99328e3 Parents: 6466cfd Author: Wes McKinney <[email protected]> Authored: Wed Sep 13 17:34:52 2017 -0400 Committer: Wes McKinney <[email protected]> Committed: Wed Sep 13 17:34:52 2017 -0400 ---------------------------------------------------------------------- CMakeLists.txt | 3 ++- README.md | 3 ++- cmake_modules/ThirdpartyToolchain.cmake | 20 +++++++++++++++++--- src/parquet/arrow/arrow-reader-writer-test.cc | 8 ++++---- src/parquet/arrow/reader.cc | 8 ++++---- src/parquet/statistics-test.cc | 10 ++++------ 6 files changed, 33 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/005767c1/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ecf877..1deab40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -499,7 +499,8 @@ if (PARQUET_BOOST_USE_SHARED) boost_shared_regex) else() set(BOOST_LINK_LIBS - boost_static_regex) + boost_static_regex + boost_static_system) endif() ############################################################# http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/005767c1/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 770f0ca..5407977 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,8 @@ Check [Windows developer guide][1] for instructions to build parquet-cpp on Wind ## Third Party Dependencies -- Apache Arrow (memory management, compression, IO, optional columnar data adapters) +- Apache Arrow >= 0.7.0 (memory management, compression, IO, optional columnar + data adapters) - Thrift 0.7+ [install instructions](https://thrift.apache.org/docs/install/) - googletest 1.7.0 (cannot be installed with package managers) - Google Benchmark (only required if building benchmarks) http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/005767c1/cmake_modules/ThirdpartyToolchain.cmake ---------------------------------------------------------------------- diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index b1ea10a..1221765 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -70,20 +70,24 @@ if (PARQUET_BOOST_USE_SHARED) add_definitions(-DBOOST_ALL_DYN_LINK) endif() - find_package(Boost COMPONENTS regex REQUIRED) + find_package(Boost COMPONENTS regex system REQUIRED) if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG") set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG}) + set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG}) else() set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE}) + set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE}) endif() else() # Find static Boost libraries. set(Boost_USE_STATIC_LIBS ON) - find_package(Boost COMPONENTS regex REQUIRED) + find_package(Boost COMPONENTS regex system REQUIRED) if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG") set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG}) + set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG}) else() set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE}) + set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE}) endif() endif() @@ -98,9 +102,19 @@ if (PARQUET_BOOST_USE_SHARED) set_target_properties(boost_shared_regex PROPERTIES IMPORTED_LOCATION "${BOOST_SHARED_REGEX_LIBRARY}") endif() + add_library(boost_shared_system SHARED IMPORTED) + if (MSVC) + set_target_properties(boost_shared_system + PROPERTIES IMPORTED_IMPLIB "${BOOST_SHARED_SYSTEM_LIBRARY}") + else() + set_target_properties(boost_shared_system + PROPERTIES IMPORTED_LOCATION "${BOOST_SHARED_SYSTEM_LIBRARY}") + endif() else() add_library(boost_static_regex STATIC IMPORTED) set_target_properties(boost_static_regex PROPERTIES IMPORTED_LOCATION ${BOOST_STATIC_REGEX_LIBRARY}) + add_library(boost_static_system STATIC IMPORTED) + set_target_properties(boost_static_system PROPERTIES IMPORTED_LOCATION ${BOOST_STATIC_SYSTEM_LIBRARY}) endif() include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) @@ -339,7 +353,7 @@ if (NOT ARROW_FOUND) -DARROW_BUILD_TESTS=OFF) if ("$ENV{PARQUET_ARROW_VERSION}" STREQUAL "") - set(ARROW_VERSION "939957f33ed0dd02013917b366ff85eb857c3947") + set(ARROW_VERSION "97f9029ce835dfc2655ca91b9820a2e6aed89107") else() set(ARROW_VERSION "$ENV{PARQUET_ARROW_VERSION}") endif() http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/005767c1/src/parquet/arrow/arrow-reader-writer-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/arrow-reader-writer-test.cc b/src/parquet/arrow/arrow-reader-writer-test.cc index 986adfc..c7ac057 100644 --- a/src/parquet/arrow/arrow-reader-writer-test.cc +++ b/src/parquet/arrow/arrow-reader-writer-test.cc @@ -779,10 +779,10 @@ TEST_F(TestUInt32ParquetIO, Parquet_1_0_Compability) { } } - const int32_t kOffset = 0; - ASSERT_OK(MakePrimitiveArray(std::make_shared<::arrow::Int64Type>(), values->length(), - int64_data, values->null_bitmap(), values->null_count(), - kOffset, &expected_values)); + std::vector<std::shared_ptr<Buffer>> buffers{values->null_bitmap(), int64_data}; + auto arr_data = std::make_shared<::arrow::ArrayData>(::arrow::int64(), values->length(), + buffers, values->null_count()); + ASSERT_OK(MakeArray(arr_data, &expected_values)); this->ReadAndCheckSingleColumnTable(expected_values); } http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/005767c1/src/parquet/arrow/reader.cc ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/reader.cc b/src/parquet/arrow/reader.cc index 9908280..8d5ea7e 100644 --- a/src/parquet/arrow/reader.cc +++ b/src/parquet/arrow/reader.cc @@ -334,7 +334,7 @@ class PARQUET_NO_EXPORT StructImpl : public ColumnReader::Impl { std::shared_ptr<Field> field_; PoolBuffer def_levels_buffer_; - Status DefLevelsToNullArray(std::shared_ptr<MutableBuffer>* null_bitmap, + Status DefLevelsToNullArray(std::shared_ptr<Buffer>* null_bitmap, int64_t* null_count); void InitField(const NodePtr& node, const std::vector<std::shared_ptr<Impl>>& children); }; @@ -1376,9 +1376,9 @@ Status ColumnReader::NextBatch(int batch_size, std::shared_ptr<Array>* out) { // StructImpl methods -Status StructImpl::DefLevelsToNullArray(std::shared_ptr<MutableBuffer>* null_bitmap_out, +Status StructImpl::DefLevelsToNullArray(std::shared_ptr<Buffer>* null_bitmap_out, int64_t* null_count_out) { - std::shared_ptr<MutableBuffer> null_bitmap; + std::shared_ptr<Buffer> null_bitmap; auto null_count = 0; ValueLevelsPtr def_levels_data; size_t def_levels_length; @@ -1460,7 +1460,7 @@ Status StructImpl::GetRepLevels(ValueLevelsPtr* data, size_t* length) { Status StructImpl::NextBatch(int batch_size, std::shared_ptr<Array>* out) { std::vector<std::shared_ptr<Array>> children_arrays; - std::shared_ptr<MutableBuffer> null_bitmap; + std::shared_ptr<Buffer> null_bitmap; int64_t null_count; // Gather children arrays and def levels http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/005767c1/src/parquet/statistics-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/statistics-test.cc b/src/parquet/statistics-test.cc index d6c5205..1521cbd 100644 --- a/src/parquet/statistics-test.cc +++ b/src/parquet/statistics-test.cc @@ -194,8 +194,9 @@ bool* TestRowGroupStatistics<BooleanType>::GetValuesPointer(std::vector<bool>& v } template <typename TestType> -typename std::vector<typename TestType::c_type> TestRowGroupStatistics< - TestType>::GetDeepCopy(const std::vector<typename TestType::c_type>& values) { +typename std::vector<typename TestType::c_type> +TestRowGroupStatistics<TestType>::GetDeepCopy( + const std::vector<typename TestType::c_type>& values) { return values; } @@ -478,7 +479,7 @@ class TestStatistics : public ::testing::Test { }; using CompareTestTypes = ::testing::Types<Int32Type, Int64Type, Int96Type, FloatType, - DoubleType, ByteArrayType, FLBAType>; + DoubleType, ByteArrayType, FLBAType>; // TYPE::INT32 template <> @@ -680,8 +681,5 @@ TEST_F(TestStatisticsFLBA, UnknownSortOrder) { ASSERT_FALSE(cc_metadata->is_stats_set()); } - - - } // namespace test } // namespace parquet
