Repository: parquet-cpp Updated Branches: refs/heads/master c2d8df9fb -> 1c4012d36
PARQUET-813: Build thirdparty dependencies using ExternalProject Author: Korn, Uwe <[email protected]> Author: Uwe L. Korn <[email protected]> Closes #207 from xhochy/PARQUET-813 and squashes the following commits: e9f57c8 [Korn, Uwe] Use arrow_io_static in unittests 6480f24 [Uwe L. Korn] Update Arrow hash 87e3acd [Uwe L. Korn] Explicitly call make install for Thrift babe81f [Korn, Uwe] Ensure external projects are built first c3d8e0e [Korn, Uwe] Remove old thirdparty scripts a9861a4 [Korn, Uwe] Fix linking problems 50d2acc [Uwe L. Korn] Add Thrift and Arrow 0e2fa19 [Korn, Uwe] PARQUET-813: Build thirdparty dependencies using ExternalProject Project: http://git-wip-us.apache.org/repos/asf/parquet-cpp/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-cpp/commit/1c4012d3 Tree: http://git-wip-us.apache.org/repos/asf/parquet-cpp/tree/1c4012d3 Diff: http://git-wip-us.apache.org/repos/asf/parquet-cpp/diff/1c4012d3 Branch: refs/heads/master Commit: 1c4012d36cee679c990f900640a8a05419df2c61 Parents: c2d8df9 Author: Korn, Uwe <[email protected]> Authored: Thu Dec 29 05:06:30 2016 -0500 Committer: Wes McKinney <[email protected]> Committed: Thu Dec 29 05:06:30 2016 -0500 ---------------------------------------------------------------------- CMakeLists.txt | 306 +++++++++++++++++++++++++++++++-- README.md | 44 +---- ci/before_script_travis.sh | 5 - cmake_modules/FindZLIB.cmake | 5 +- setup_build_env.sh | 42 ----- src/parquet/arrow/CMakeLists.txt | 8 +- src/parquet/thrift/CMakeLists.txt | 2 +- thirdparty/build_thirdparty.sh | 163 ------------------ thirdparty/download_thirdparty.sh | 73 -------- thirdparty/set_thirdparty_env.sh | 37 ---- thirdparty/versions.sh | 44 ----- 11 files changed, 310 insertions(+), 419 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/1c4012d3/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index a9fe089..79e43f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,8 @@ cmake_minimum_required(VERSION 2.6) project(parquet-cpp) +include(ExternalProject) + if (NOT "$ENV{PARQUET_GCC_ROOT}" STREQUAL "") set(GCC_ROOT $ENV{PARQUET_GCC_ROOT}) set(CMAKE_C_COMPILER ${GCC_ROOT}/bin/gcc) @@ -266,6 +268,11 @@ enable_testing() # Dependencies ############################################################ +set(GTEST_VERSION "1.7.0") +set(GBENCHMARK_VERSION "1.0.0") +set(SNAPPY_VERSION "1.1.3") +set(THRIFT_VERSION "0.9.1") + # find boost headers and libs set(Boost_DEBUG TRUE) set(Boost_USE_MULTITHREADED ON) @@ -276,26 +283,131 @@ message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIRS}) message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) # find thrift headers and libs -find_package(Thrift REQUIRED) +find_package(Thrift) + +if (NOT THRIFT_FOUND) + if (APPLE) + message(FATAL_ERROR "thrift compilation under OSX is not currently supported.") + endif() + + set(THRIFT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/thrift_ep/src/thrift_ep-install") + set(THRIFT_HOME "${THRIFT_PREFIX}") + set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include") + set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/libthrift.a") + set(THRIFT_COMPILER "${THRIFT_PREFIX}/bin/thrift") + set(THRIFT_VENDORED 1) + + if (CMAKE_VERSION VERSION_GREATER "3.2") + # BUILD_BYPRODUCTS is a 3.2+ feature + ExternalProject_Add(thrift_ep + CONFIGURE_COMMAND ./configure "CXXFLAGS=-fPIC" --without-qt4 --without-c_glib --without-csharp --without-java --without-erlang --without-nodejs --without-lua --without-python --without-perl --without-php --without-php_extension --without-ruby --without-haskell --without-go --without-d --with-cpp "--prefix=${THRIFT_PREFIX}" + BUILD_IN_SOURCE 1 + # This is needed for 0.9.1 and can be removed for 0.9.3 again + BUILD_COMMAND make clean + INSTALL_COMMAND make install + INSTALL_DIR ${THRIFT_PREFIX} + URL "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz"; + BUILD_BYPRODUCTS "${THRIFT_STATIC_LIB}" "${THRIFT_COMPILER}" + ) + else() + ExternalProject_Add(thrift_ep + CONFIGURE_COMMAND ./configure "CXXFLAGS=-fPIC" --without-qt4 --without-c_glib --without-csharp --without-java --without-erlang --without-nodejs --without-lua --without-python --without-perl --without-php --without-php_extension --without-ruby --without-haskell --without-go --without-d --with-cpp "--prefix=${THRIFT_PREFIX}" + BUILD_IN_SOURCE 1 + # This is needed for 0.9.1 and can be removed for 0.9.3 again + BUILD_COMMAND make clean + INSTALL_COMMAND make install + INSTALL_DIR ${THRIFT_PREFIX} + URL "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz"; + ) + endif() + set(THRIFT_VENDORED 1) +else() + set(THRIFT_VENDORED 0) +endif() + include_directories(SYSTEM ${THRIFT_INCLUDE_DIR} ${THRIFT_INCLUDE_DIR}/thrift) -set(LIBS ${LIBS} ${THRIFT_LIBS}) message(STATUS "Thrift include dir: ${THRIFT_INCLUDE_DIR}") -message(STATUS "Thrift contrib dir: ${THRIFT_CONTRIB_DIR}") -message(STATUS "Thrift library path: ${THRIFT_LIBS}") message(STATUS "Thrift static library: ${THRIFT_STATIC_LIB}") message(STATUS "Thrift compiler: ${THRIFT_COMPILER}") -# for static linking with Thrift, THRIFT_STATIC_LIB is set in FindThrift.cmake add_library(thriftstatic STATIC IMPORTED) set_target_properties(thriftstatic PROPERTIES IMPORTED_LOCATION ${THRIFT_STATIC_LIB}) +if (THRIFT_VENDORED) + add_dependencies(thriftstatic thrift_ep) +endif() + ## Snappy -find_package(Snappy REQUIRED) +find_package(Snappy) +if (NOT SNAPPY_FOUND) + set(SNAPPY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/snappy_ep/src/snappy_ep-install") + set(SNAPPY_HOME "${SNAPPY_PREFIX}") + set(SNAPPY_INCLUDE_DIR "${SNAPPY_PREFIX}/include") + set(SNAPPY_STATIC_LIB "${SNAPPY_PREFIX}/lib/libsnappy.a") + set(SNAPPY_VENDORED 1) + + if (CMAKE_VERSION VERSION_GREATER "3.2") + # BUILD_BYPRODUCTS is a 3.2+ feature + ExternalProject_Add(snappy_ep + CONFIGURE_COMMAND ./configure --with-pic "--prefix=${SNAPPY_PREFIX}" + BUILD_IN_SOURCE 1 + BUILD_COMMAND ${MAKE} + INSTALL_DIR ${SNAPPY_PREFIX} + URL "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz"; + BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}" + ) + else() + ExternalProject_Add(snappy_ep + CONFIGURE_COMMAND ./configure --with-pic "--prefix=${SNAPPY_PREFIX}" + BUILD_IN_SOURCE 1 + BUILD_COMMAND ${MAKE} + INSTALL_DIR ${SNAPPY_PREFIX} + URL "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz"; + ) + endif() +else() + set(SNAPPY_VENDORED 0) +endif() + include_directories(SYSTEM ${SNAPPY_INCLUDE_DIR}) add_library(snappystatic STATIC IMPORTED) set_target_properties(snappystatic PROPERTIES IMPORTED_LOCATION ${SNAPPY_STATIC_LIB}) +if (SNAPPY_VENDORED) + add_dependencies(snappystatic snappy_ep) +endif() + ## Brotli -find_package(Brotli REQUIRED) +find_package(Brotli) +if (NOT BROTLI_FOUND) + set(BROTLI_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/brotli_ep/src/brotli_ep-install") + set(BROTLI_HOME "${BROTLI_PREFIX}") + set(BROTLI_INCLUDE_DIR "${BROTLI_PREFIX}/include") + set(BROTLI_LIBRARY_ENC "${BROTLI_PREFIX}/lib/${CMAKE_LIBRARY_ARCHITECTURE}/libbrotlienc.a") + set(BROTLI_LIBRARY_DEC "${BROTLI_PREFIX}/lib/${CMAKE_LIBRARY_ARCHITECTURE}/libbrotlidec.a") + set(BROTLI_LIBRARY_COMMON "${BROTLI_PREFIX}/lib/${CMAKE_LIBRARY_ARCHITECTURE}/libbrotlicommon.a") + set(BROTLI_VENDORED 1) + set(BROTLI_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX} + -DCMAKE_INSTALL_LIBDIR=lib/${CMAKE_LIBRARY_ARCHITECTURE} + -DBUILD_SHARED_LIBS=OFF) + + if (CMAKE_VERSION VERSION_GREATER "3.2") + # BUILD_BYPRODUCTS is a 3.2+ feature + ExternalProject_Add(brotli_ep + GIT_REPOSITORY https://github.com/google/brotli.git + GIT_TAG 5db62dcc9d386579609540cdf8869e95ad334bbd + BUILD_BYPRODUCTS "${BROTLI_LIBRARY_ENC}" "${BROTLI_LIBRARY_DEC}" "${BROTLI_LIBRARY_COMMON}" + CMAKE_ARGS ${BROTLI_CMAKE_ARGS}) + else() + ExternalProject_Add(brotli_ep + GIT_REPOSITORY https://github.com/google/brotli.git + GIT_TAG 5db62dcc9d386579609540cdf8869e95ad334bbd + CMAKE_ARGS ${BROTLI_CMAKE_ARGS}) + endif() +else() + set(BROTLI_VENDORED 0) +endif() + include_directories(SYSTEM ${BROTLI_INCLUDE_DIR}) add_library(brotlistatic_enc STATIC IMPORTED) set_target_properties(brotlistatic_enc PROPERTIES IMPORTED_LOCATION ${BROTLI_LIBRARY_ENC}) @@ -304,19 +416,101 @@ set_target_properties(brotlistatic_dec PROPERTIES IMPORTED_LOCATION ${BROTLI_LIB add_library(brotlistatic_common STATIC IMPORTED) set_target_properties(brotlistatic_common PROPERTIES IMPORTED_LOCATION ${BROTLI_LIBRARY_COMMON}) +if (BROTLI_VENDORED) + add_dependencies(brotlistatic_enc brotli_ep) + add_dependencies(brotlistatic_dec brotli_ep) + add_dependencies(brotlistatic_common brotli_ep) +endif() + ## ZLIB -find_package(ZLIB REQUIRED) +# For now: Always build zlib so that the shared lib has -fPIC +# find_package(ZLIB) + +if (NOT ZLIB_FOUND) + set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep/src/zlib_ep-install") + set(ZLIB_HOME "${ZLIB_PREFIX}") + set(ZLIB_INCLUDE_DIR "${ZLIB_PREFIX}/include") + set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/libz.a") + set(ZLIB_VENDORED 1) + set(ZLIB_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX} + -DCMAKE_C_FLAGS=-fPIC + -DBUILD_SHARED_LIBS=OFF) + + if (CMAKE_VERSION VERSION_GREATER "3.2") + # BUILD_BYPRODUCTS is a 3.2+ feature + ExternalProject_Add(zlib_ep + URL "http://zlib.net/zlib-1.2.8.tar.gz"; + BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}" + CMAKE_ARGS ${ZLIB_CMAKE_ARGS}) + else() + ExternalProject_Add(zlib_ep + URL "http://zlib.net/zlib-1.2.8.tar.gz"; + CMAKE_ARGS ${ZLIB_CMAKE_ARGS}) + endif() +else() + set(ZLIB_VENDORED 0) +endif() + include_directories(SYSTEM ${ZLIB_INCLUDE_DIRS}) add_library(zlibstatic STATIC IMPORTED) set_target_properties(zlibstatic PROPERTIES IMPORTED_LOCATION ${ZLIB_STATIC_LIB}) +if (ZLIB_VENDORED) + add_dependencies(zlibstatic zlib_ep) +endif() + ## GTest if(PARQUET_BUILD_TESTS) add_custom_target(unittest ctest -L unittest) - find_package(GTest REQUIRED) + + if("$ENV{GTEST_HOME}" STREQUAL "") + if(APPLE) + set(GTEST_CMAKE_CXX_FLAGS "-fPIC -std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes") + else() + set(GTEST_CMAKE_CXX_FLAGS "-fPIC") + endif() + + set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") + set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") + set(GTEST_STATIC_LIB "${GTEST_PREFIX}/${CMAKE_CFG_INTDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GTEST_VENDORED 1) + + if (CMAKE_VERSION VERSION_GREATER "3.2") + # BUILD_BYPRODUCTS is a 3.2+ feature + ExternalProject_Add(googletest_ep + URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"; + CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} -Dgtest_force_shared_crt=ON + # googletest doesn't define install rules, so just build in the + # source dir and don't try to install. See its README for + # details. + BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS "${GTEST_STATIC_LIB}" + INSTALL_COMMAND "") + else() + ExternalProject_Add(googletest_ep + URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"; + CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} -Dgtest_force_shared_crt=ON + # googletest doesn't define install rules, so just build in the + # source dir and don't try to install. See its README for + # details. + BUILD_IN_SOURCE 1 + INSTALL_COMMAND "") + endif() + else() + find_package(GTest REQUIRED) + set(GTEST_VENDORED 0) + endif() + + message(STATUS "GTest include dir: ${GTEST_INCLUDE_DIR}") + message(STATUS "GTest static library: ${GTEST_STATIC_LIB}") include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) add_library(gtest STATIC IMPORTED) set_target_properties(gtest PROPERTIES IMPORTED_LOCATION ${GTEST_STATIC_LIB}) + + if(GTEST_VENDORED) + add_dependencies(gtest googletest_ep) + endif() endif() ## Google Benchmark @@ -326,11 +520,47 @@ endif() if(PARQUET_BUILD_BENCHMARKS) add_custom_target(runbenchmark ctest -L benchmark) - find_package(GBenchmark REQUIRED) + + if("$ENV{GBENCHMARK_HOME}" STREQUAL "") + if(APPLE) + set(GBENCHMARK_CMAKE_CXX_FLAGS "-std=c++11 -stdlib=libc++") + else() + set(GBENCHMARK_CMAKE_CXX_FLAGS "--std=c++11") + endif() + + set(GBENCHMARK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gbenchmark_ep/src/gbenchmark_ep-install") + set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include") + set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GBENCHMARK_VENDORED 1) + set(GBENCHMARK_CMAKE_ARGS + "-DCMAKE_BUILD_TYPE=Release" + "-DCMAKE_INSTALL_PREFIX:PATH=${GBENCHMARK_PREFIX}" + "-DCMAKE_CXX_FLAGS=-fPIC ${GBENCHMARK_CMAKE_CXX_FLAGS}") + if (CMAKE_VERSION VERSION_GREATER "3.2") + # BUILD_BYPRODUCTS is a 3.2+ feature + ExternalProject_Add(gbenchmark_ep + URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"; + BUILD_BYPRODUCTS "${GBENCHMARK_STATIC_LIB}" + CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS}) + else() + ExternalProject_Add(gbenchmark_ep + URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"; + CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS}) + endif() + else() + find_package(GBenchmark REQUIRED) + set(GBENCHMARK_VENDORED 0) + endif() + + message(STATUS "GBenchmark include dir: ${GBENCHMARK_INCLUDE_DIR}") + message(STATUS "GBenchmark static library: ${GBENCHMARK_STATIC_LIB}") include_directories(SYSTEM ${GBENCHMARK_INCLUDE_DIR}) - message(${GBENCHMARK_STATIC_LIB}) add_library(gbenchmark STATIC IMPORTED) set_target_properties(gbenchmark PROPERTIES IMPORTED_LOCATION ${GBENCHMARK_STATIC_LIB}) + + if(GBENCHMARK_VENDORED) + add_dependencies(gbenchmark gbenchmark_ep) + endif() endif() # Thrift requires these definitions for some types that we use @@ -562,6 +792,10 @@ add_library(parquet_objlib OBJECT ${LIBPARQUET_SRCS} ) +# Although we don't link parquet_objlib against anything, we need it to depend +# on these libs as we may generate their headers via ExternalProject_Add +add_dependencies(parquet_objlib ${LIBPARQUET_PRIVATE_LINK_LIBS}) + set_property(TARGET parquet_objlib PROPERTY POSITION_INDEPENDENT_CODE 1) if(APPLE) @@ -622,7 +856,48 @@ add_subdirectory(tools) # Arrow if (PARQUET_ARROW) - find_package(Arrow REQUIRED) + + set(ARROW_VERSION "4733ee876e1fddb8032fce1dc9e486d68904fbea") + + find_package(Arrow) + + if (NOT ARROW_FOUND) + set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep/src/arrow_ep-install") + set(ARROW_HOME "${ARROW_PREFIX}") + set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") + set(ARROW_SHARED_LIB "${ARROW_PREFIX}/lib/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX}") + set(ARROW_IO_SHARED_LIB "${ARROW_PREFIX}/lib/libarrow_io${CMAKE_SHARED_LIBRARY_SUFFIX}") + set(ARROW_STATIC_LIB "${ARROW_PREFIX}/lib/libarrow.a") + set(ARROW_IO_STATIC_LIB "${ARROW_PREFIX}/lib/libarrow_io.a") + set(ARROW_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX} + -DARROW_BUILD_TESTS=OFF + -DARROW_HDFS=ON) + + if (CMAKE_VERSION VERSION_GREATER "3.2") + # BUILD_BYPRODUCTS is a 3.2+ feature + ExternalProject_Add(arrow_ep + GIT_REPOSITORY https://github.com/apache/arrow.git + GIT_TAG ${ARROW_VERSION} + BUILD_BYPRODUCTS "${ARROW_SHARED_LIB}" "${ARROW_IO_SHARED_LIB}" "${ARROW_IO_STATIC_LIB}" "${ARROW_STATIC_LIB}" + # With CMake 3.7.0 there is a SOURCE_SUBDIR argument which we can use + # to specify that the CMakeLists.txt of Arrow is located in cpp/ + # + # See https://gitlab.kitware.com/cmake/cmake/commit/a8345d65f359d75efb057d22976cfb92b4d477cf + CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp + CMAKE_ARGS ${ARROW_CMAKE_ARGS}) + else() + ExternalProject_Add(arrow_ep + GIT_REPOSITORY https://github.com/apache/arrow.git + GIT_TAG ${ARROW_VERSION} + CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp + CMAKE_ARGS ${ARROW_CMAKE_ARGS}) + endif() + set(ARROW_VENDORED 1) + else() + set(ARROW_VENDORED 0) + endif() + include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) add_library(arrow SHARED IMPORTED) set_target_properties(arrow PROPERTIES IMPORTED_LOCATION ${ARROW_SHARED_LIB}) @@ -633,6 +908,13 @@ if (PARQUET_ARROW) add_library(arrow_io_static STATIC IMPORTED) set_target_properties(arrow_io_static PROPERTIES IMPORTED_LOCATION ${ARROW_IO_STATIC_LIB}) + if (ARROW_VENDORED) + add_dependencies(arrow arrow_ep) + add_dependencies(arrow_io arrow_ep) + add_dependencies(arrow_static arrow_ep) + add_dependencies(arrow_io_static arrow_ep) + endif() + add_subdirectory(src/parquet/arrow) endif() http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/1c4012d3/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 2dc023c..54613c1 100644 --- a/README.md +++ b/README.md @@ -39,48 +39,14 @@ - googletest 1.7.0 (cannot be installed with package managers) - Google Benchmark (only required if building benchmarks) -You can install these dependencies using a package manager or using the -`thirdparty/` scripts in this repository. +You can either install these dependencies via your package manager, otherwise +they will be build automatically as part of the build. -Mac already has zlib; on Homebrew, you can run: - -```shell - brew install snappy thrift -``` - -For Linux, just run - -```shell -source setup_build_env.sh -``` - -By default, it will create a build directory `build/`. You can override the -build directory by setting the BUILD_DIR env variable to another location. - -To manually build the thirdparty libraries in-tree, run: - -```shell -./thirdparty/download_thirdparty.sh -./thirdparty/build_thirdparty.sh -source thirdparty/set_thirdparty_env.sh -``` - -After building the thirdparty libraries, for future development iteration you -can set the dependency environment variables (detailed below) by running - -```shell -source $BUILD_DIR/thirdparty/set_thirdparty_env.sh -``` - -The unit tests depend on `googletest` which cannot be installed with Homebrew -or normal package managers. If you wish to use system dependencies, we -recommend that you build googletest in-tree by running: +Note that thrift will not be build inside the project on macOS. Instead you +should install it via homebrew: ``` -./thirdparty/download_thirdparty.sh -./thirdparty/build_thirdparty.sh gtest -source thirdparty/versions.sh -export GTEST_HOME=`pwd`/thirdparty/$GTEST_BASEDIR +brew install thrift ``` ## Build http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/1c4012d3/ci/before_script_travis.sh ---------------------------------------------------------------------- diff --git a/ci/before_script_travis.sh b/ci/before_script_travis.sh index 746e10f..75679fb 100755 --- a/ci/before_script_travis.sh +++ b/ci/before_script_travis.sh @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. -# Build an isolated thirdparty -cp -r $TRAVIS_BUILD_DIR/thirdparty . -./thirdparty/download_thirdparty.sh - if [ $TRAVIS_OS_NAME == "osx" ]; then brew update > /dev/null brew install thrift @@ -26,4 +22,3 @@ else fi export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/data -source $TRAVIS_BUILD_DIR/setup_build_env.sh http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/1c4012d3/cmake_modules/FindZLIB.cmake ---------------------------------------------------------------------- diff --git a/cmake_modules/FindZLIB.cmake b/cmake_modules/FindZLIB.cmake index b0c11db..6653046 100644 --- a/cmake_modules/FindZLIB.cmake +++ b/cmake_modules/FindZLIB.cmake @@ -44,12 +44,13 @@ if ( _zlib_roots ) find_path( ZLIB_INCLUDE_DIR NAMES zlib.h PATHS ${_zlib_roots} NO_DEFAULT_PATH PATH_SUFFIXES "include" ) - find_library( ZLIB_LIBRARIES NAMES z + find_library( ZLIB_LIBRARIES NAMES libz.a PATHS ${_zlib_roots} NO_DEFAULT_PATH PATH_SUFFIXES "lib" ) else () find_path( ZLIB_INCLUDE_DIR NAMES zlib.h ) - find_library( ZLIB_LIBRARIES NAMES z ) + # Only look for the static library + find_library( ZLIB_LIBRARIES NAMES libz.a ) endif () http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/1c4012d3/setup_build_env.sh ---------------------------------------------------------------------- diff --git a/setup_build_env.sh b/setup_build_env.sh deleted file mode 100755 index eabe6a5..0000000 --- a/setup_build_env.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -: ${BUILD_DIR:=$SOURCE_DIR/build} - -# Create an isolated thirdparty -mkdir -p $BUILD_DIR -cp -r $SOURCE_DIR/thirdparty $BUILD_DIR -pushd $BUILD_DIR - -./thirdparty/download_thirdparty.sh -./thirdparty/build_thirdparty.sh -source thirdparty/set_thirdparty_env.sh - -export PARQUET_TEST_DATA=$SOURCE_DIR/data - -# Thrift needs to be installed from brew on OS X -if [ "$(uname)" == "Darwin" ]; then - export -n THRIFT_HOME -fi - -popd - -echo -echo "Build env initialized in $BUILD_DIR." http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/1c4012d3/src/parquet/arrow/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/CMakeLists.txt b/src/parquet/arrow/CMakeLists.txt index 376e84a..37e4894 100644 --- a/src/parquet/arrow/CMakeLists.txt +++ b/src/parquet/arrow/CMakeLists.txt @@ -29,6 +29,12 @@ add_library(parquet_arrow_objlib OBJECT ${PARQUET_ARROW_SRCS} ) +# Add dependencies so ExternalProjects are built beforehand +add_dependencies(parquet_arrow_objlib + arrow_static + arrow_io_static + parquet_static) + # SET_TARGET_PROPERTIES(parquet_arrow PROPERTIES LINKER_LANGUAGE CXX) if (PARQUET_BUILD_SHARED) @@ -62,7 +68,7 @@ if (PARQUET_BUILD_STATIC) OUTPUT_NAME "parquet_arrow") target_link_libraries(parquet_arrow_static arrow_static - arrow_io + arrow_io_static parquet_static) install(TARGETS parquet_arrow_static ARCHIVE DESTINATION lib http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/1c4012d3/src/parquet/thrift/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/parquet/thrift/CMakeLists.txt b/src/parquet/thrift/CMakeLists.txt index 1c38702..78d9cd7 100644 --- a/src/parquet/thrift/CMakeLists.txt +++ b/src/parquet/thrift/CMakeLists.txt @@ -43,7 +43,7 @@ get_filename_component(ABS_PARQUET_THRIFT parquet.thrift ABSOLUTE) add_custom_command( OUTPUT ${THRIFT_OUTPUT_FILES} COMMAND ${THRIFT_COMPILER} --gen cpp -out ${OUTPUT_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/parquet.thrift - DEPENDS ${ABS_PARQUET_THRIFT} + DEPENDS ${ABS_PARQUET_THRIFT} thriftstatic COMMENT "Running thrift compiler on parquet.thrift" VERBATIM ) http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/1c4012d3/thirdparty/build_thirdparty.sh ---------------------------------------------------------------------- diff --git a/thirdparty/build_thirdparty.sh b/thirdparty/build_thirdparty.sh deleted file mode 100755 index 6eb838b..0000000 --- a/thirdparty/build_thirdparty.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -x -set -e -TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -source $TP_DIR/versions.sh -PREFIX=$TP_DIR/installed - -################################################################################ - -if [ "$#" = "0" ]; then - F_ALL=1 -else - # Allow passing specific libs to build on the command line - for arg in "$@"; do - case $arg in - "arrow") F_ARROW=1 ;; - "brotli") F_BROTLI=1 ;; - "zlib") F_ZLIB=1 ;; - "gbenchmark") F_GBENCHMARK=1 ;; - "gtest") F_GTEST=1 ;; - "snappy") F_SNAPPY=1 ;; - "thrift") F_THRIFT=1 ;; - *) echo "Unknown module: $arg"; exit 1 ;; - esac - done -fi - -################################################################################ - -# Determine how many parallel jobs to use for make based on the number of cores -if [[ "$OSTYPE" =~ ^linux ]]; then - PARALLEL=$(grep -c processor /proc/cpuinfo) -elif [[ "$OSTYPE" == "darwin"* ]]; then - PARALLEL=$(sysctl -n hw.ncpu) -else - echo Unsupported platform $OSTYPE - exit 1 -fi - -mkdir -p "$PREFIX/include" -mkdir -p "$PREFIX/lib" - -# On some systems, autotools installs libraries to lib64 rather than lib. Fix -# this by setting up lib64 as a symlink to lib. We have to do this step first -# to handle cases where one third-party library depends on another. -ln -sf lib "$PREFIX/lib64" - -# use the compiled tools -export PATH=$PREFIX/bin:$PATH - -# build snappy -if [ -n "$F_ALL" -o -n "$F_SNAPPY" ]; then - cd $TP_DIR/$SNAPPY_BASEDIR - ./configure --with-pic --prefix=$PREFIX - make -j$PARALLEL install -fi - -STANDARD_DARWIN_FLAGS="-std=c++11 -stdlib=libc++" - -# build googletest -GOOGLETEST_ERROR="failed for googletest!" -if [ -n "$F_ALL" -o -n "$F_GTEST" ]; then - cd $TP_DIR/$GTEST_BASEDIR - - if [[ "$OSTYPE" == "darwin"* ]]; then - CXXFLAGS=-fPIC cmake -DCMAKE_CXX_FLAGS="$STANDARD_DARWIN_FLAGS -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes" || { echo "cmake $GOOGLETEST_ERROR" ; exit 1; } - else - CXXFLAGS=-fPIC cmake . || { echo "cmake $GOOGLETEST_ERROR"; exit 1; } - fi - - make VERBOSE=1 || { echo "Make $GOOGLETEST_ERROR" ; exit 1; } -fi - -# build google benchmark -GBENCHMARK_ERROR="failed for google benchmark" -if [ -n "$F_ALL" -o -n "$F_GBENCHMARK" ]; then - cd $TP_DIR/$GBENCHMARK_BASEDIR - - CMAKE_CXX_FLAGS="--std=c++11" - if [[ "$OSTYPE" == "darwin"* ]]; then - CMAKE_CXX_FLAGS=$STANDARD_DARWIN_FLAGS - fi - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PREFIX -DCMAKE_CXX_FLAGS="-fPIC $CMAKE_CXX_FLAGS" . || { echo "cmake $GBENCHMARK_ERROR" ; exit 1; } - - make VERBOSE=1 install || { echo "make $GBENCHMARK_ERROR" ; exit 1; } -fi - -# build zlib -if [ -n "$F_ALL" -o -n "$F_ZLIB" ]; then - cd $TP_DIR/$ZLIB_BASEDIR - CFLAGS=-fPIC cmake -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX $ZLIB_DIR - make -j$PARALLEL install -fi - -# build thrift -if [ -n "$F_ALL" -o -n "$F_THRIFT" ]; then - if [ "$(uname)" == "Darwin" ]; then - echo "thrift compilation under OSX is not currently supported." - - # exit with an error if thrift was specified explicitly otherwise it is - # just a warning - if [ -n "$F_THRIFT" ]; then - exit 1 - fi - else - # linux build - # this expects all of the depedencies for thrift to already be installed in - # such a way that ./configure can find them - cd $TP_DIR/$THRIFT_BASEDIR - ./configure CXXFLAGS='-fPIC' --without-qt4 --without-c_glib --without-csharp --without-java --without-erlang --without-nodejs --without-lua --without-python --without-perl --without-php --without-php_extension --without-ruby --without-haskell --without-go --without-d --with-cpp --prefix=$PREFIX - - # This must be removed with Thrift 0.9.3, but required for Thrift 0.9.1 - make clean - make install - fi -fi - -# build arrow -if [ -n "$F_ALL" -o -n "$F_ARROW" ]; then - cd $TP_DIR/$ARROW_BASEDIR/cpp - cmake -DARROW_BUILD_TESTS=off \ - -DARROW_HDFS=ON \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - . - make -j$PARALLEL install - # : -fi - -# build brotli -if [ -n "$F_ALL" -o -n "$F_BROTLI" ]; then - cd $TP_DIR/$BROTLI_BASEDIR - cmake -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DCMAKE_INSTALL_LIBDIR="lib" \ - -DBUILD_SHARED_LIBS=OFF . - make -j$PARALLEL install - cmake -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DCMAKE_INSTALL_LIBDIR="lib" \ - -DBUILD_SHARED_LIBS=on . - make -j$PARALLEL install - # : -fi - -echo "---------------------" -echo "Thirdparty dependencies built and installed into $PREFIX successfully" http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/1c4012d3/thirdparty/download_thirdparty.sh ---------------------------------------------------------------------- diff --git a/thirdparty/download_thirdparty.sh b/thirdparty/download_thirdparty.sh deleted file mode 100755 index 4831bbf..0000000 --- a/thirdparty/download_thirdparty.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -x -set -e - -TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -source $TP_DIR/versions.sh - -: ${PARQUET_INSECURE_CURL=0} - -download_extract_and_cleanup() { - filename=$TP_DIR/$(basename "$1") - if [ "$PARQUET_INSECURE_CURL" == "1" ]; then - curl -L -k "$1" -o $filename - else - curl -#LC - "$1" -o $filename - fi - tar xzf $filename -C $TP_DIR - rm $filename -} - -if [ ! -d ${ARROW_BASEDIR} ]; then - echo "Fetching arrow" - download_extract_and_cleanup $ARROW_URL -fi - -if [ ! -d ${BROTLI_BASEDIR} ]; then - echo "Fetching brotli" - download_extract_and_cleanup $BROTLI_URL -fi - -if [ ! -d ${SNAPPY_BASEDIR} ]; then - echo "Fetching snappy" - download_extract_and_cleanup $SNAPPY_URL -fi - -if [ ! -d ${GTEST_BASEDIR} ]; then - echo "Fetching gtest" - download_extract_and_cleanup $GTEST_URL -fi - -if [ ! -d ${GBENCHMARK_BASEDIR} ]; then - echo "Fetching gtest" - download_extract_and_cleanup $GBENCHMARK_URL -fi - -if [ ! -d ${THRIFT_BASEDIR} ]; then - echo "Fetching thrift" - download_extract_and_cleanup $THRIFT_URL -fi - -if [ ! -d ${ZLIB_BASEDIR} ]; then - echo "Fetching zlib" - download_extract_and_cleanup $ZLIB_URL -fi http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/1c4012d3/thirdparty/set_thirdparty_env.sh ---------------------------------------------------------------------- diff --git a/thirdparty/set_thirdparty_env.sh b/thirdparty/set_thirdparty_env.sh deleted file mode 100644 index e8a6068..0000000 --- a/thirdparty/set_thirdparty_env.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -source $SOURCE_DIR/versions.sh - -if [ -z "$THIRDPARTY_DIR" ]; then - THIRDPARTY_DIR=$SOURCE_DIR -fi - -export ARROW_HOME=$THIRDPARTY_DIR/installed -export BROTLI_HOME=$THIRDPARTY_DIR/installed -export SNAPPY_HOME=$THIRDPARTY_DIR/installed -export ZLIB_HOME=$THIRDPARTY_DIR/installed -# build script doesn't support building thrift on OSX -if [ "$(uname)" != "Darwin" ]; then - export THRIFT_HOME=$THIRDPARTY_DIR/installed -fi - -export GTEST_HOME=$THIRDPARTY_DIR/$GTEST_BASEDIR -export GBENCHMARK_HOME=$THIRDPARTY_DIR/installed http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/1c4012d3/thirdparty/versions.sh ---------------------------------------------------------------------- diff --git a/thirdparty/versions.sh b/thirdparty/versions.sh deleted file mode 100755 index 4233669..0000000 --- a/thirdparty/versions.sh +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARROW_VERSION="268ffbeffb1cd0617e52d381d500a2d10f61124c" -ARROW_URL="https://github.com/apache/arrow/archive/${ARROW_VERSION}.tar.gz"; -ARROW_BASEDIR="arrow-${ARROW_VERSION}" - -BROTLI_VERSION="5db62dcc9d386579609540cdf8869e95ad334bbd" -BROTLI_URL="https://github.com/google/brotli/archive/${BROTLI_VERSION}.tar.gz"; -BROTLI_BASEDIR="brotli-${BROTLI_VERSION}" - -SNAPPY_VERSION=1.1.3 -SNAPPY_URL="https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz"; -SNAPPY_BASEDIR=snappy-$SNAPPY_VERSION - -THRIFT_VERSION=0.9.1 -THRIFT_URL="http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz"; -THRIFT_BASEDIR=thrift-$THRIFT_VERSION - -GBENCHMARK_VERSION=1.0.0 -GBENCHMARK_URL="https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz"; -GBENCHMARK_BASEDIR=benchmark-$GBENCHMARK_VERSION - -GTEST_VERSION=1.7.0 -GTEST_URL="https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"; -GTEST_BASEDIR=googletest-release-$GTEST_VERSION - -ZLIB_VERSION=1.2.8 -ZLIB_URL=http://zlib.net/zlib-${ZLIB_VERSION}.tar.gz -ZLIB_BASEDIR=zlib-${ZLIB_VERSION}
