Repository: parquet-cpp Updated Branches: refs/heads/master 5c02c7583 -> 562651e11
PARQUET-1048: Apache Arrow static transitive dependencies Author: Deepak Majeti <[email protected]> Closes #367 from majetideepak/PARQUET-1048 and squashes the following commits: 42635f2 [Deepak Majeti] try another script bb96b28 [Deepak Majeti] disable boost shared b3a0e75 [Deepak Majeti] enable mem check db50b65 [Deepak Majeti] build arrow shared only if parquet shared is specified 5fe857e [Deepak Majeti] Remove mem check 68f8c59 [Deepak Majeti] Fix Arrow visibility warnings 0bbd073 [Deepak Majeti] add env vars e4406ab [Deepak Majeti] Add static linking for travis ci 032c8c7 [Deepak Majeti] Fix linking of benchmarks and examples bdb5b2f [Deepak Majeti] Add Arrow lib transitive dependencies Project: http://git-wip-us.apache.org/repos/asf/parquet-cpp/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-cpp/commit/562651e1 Tree: http://git-wip-us.apache.org/repos/asf/parquet-cpp/tree/562651e1 Diff: http://git-wip-us.apache.org/repos/asf/parquet-cpp/diff/562651e1 Branch: refs/heads/master Commit: 562651e11a3a148ead4a27c5ab5ede903f1fd5bb Parents: 5c02c75 Author: Deepak Majeti <[email protected]> Authored: Mon Jul 10 19:40:10 2017 +0200 Committer: Uwe L. Korn <[email protected]> Committed: Mon Jul 10 19:40:10 2017 +0200 ---------------------------------------------------------------------- .travis.yml | 6 +++ CMakeLists.txt | 39 +++++++++++++- benchmarks/CMakeLists.txt | 5 +- ci/travis_script_static.sh | 76 ++++++++++++++++++++++++++++ cmake_modules/ThirdpartyToolchain.cmake | 1 + examples/CMakeLists.txt | 2 +- src/parquet/arrow/reader.cc | 4 +- 7 files changed, 127 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index b332cf1..70da9ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,6 +45,12 @@ matrix: env: PARQUET_BUILD_GROUP=toolchain before_script: script: + - $TRAVIS_BUILD_DIR/ci/travis_script_static.sh + - compiler: gcc + os: linux + env: PARQUET_BUILD_GROUP=toolchain + before_script: + script: - $TRAVIS_BUILD_DIR/ci/travis_script_toolchain.sh - compiler: clang os: linux http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index 5db2a54..b7a41d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -509,8 +509,45 @@ if ("${PARQUET_ARROW_LINKAGE}" STREQUAL "shared") set(ARROW_LINK_LIBS arrow) else() + ############################################################# + # Transitive Library Linkage + + if (NOT DEFINED ENV{BROTLI_STATIC_LIB_ENC} OR + NOT DEFINED ENV{BROTLI_STATIC_LIB_DEC} OR + NOT DEFINED ENV{BROTLI_STATIC_LIB_COMMON} OR + NOT DEFINED ENV{SNAPPY_STATIC_LIB} OR + NOT DEFINED ENV{ZLIB_STATIC_LIB}) + message(FATAL_ERROR "Missing transitive dependencies for Arrow static linking") + endif() + + set(BROTLI_STATIC_LIB_ENC "$ENV{BROTLI_STATIC_LIB_ENC}") + set(BROTLI_STATIC_LIB_DEC "$ENV{BROTLI_STATIC_LIB_DEC}") + set(BROTLI_STATIC_LIB_COMMON "$ENV{BROTLI_STATIC_LIB_COMMON}") + set(SNAPPY_STATIC_LIB "$ENV{SNAPPY_STATIC_LIB}") + set(ZLIB_STATIC_LIB "$ENV{ZLIB_STATIC_LIB}") + + add_library(brotli_enc STATIC IMPORTED) + set_target_properties(brotli_enc PROPERTIES IMPORTED_LOCATION ${BROTLI_STATIC_LIB_ENC}) + add_library(brotli_dec STATIC IMPORTED) + set_target_properties(brotli_dec PROPERTIES IMPORTED_LOCATION ${BROTLI_STATIC_LIB_DEC}) + add_library(brotli_common STATIC IMPORTED) + set_target_properties(brotli_common PROPERTIES IMPORTED_LOCATION ${BROTLI_STATIC_LIB_COMMON}) + add_library(snappy STATIC IMPORTED) + set_target_properties(snappy PROPERTIES IMPORTED_LOCATION ${SNAPPY_STATIC_LIB}) + add_library(zlib STATIC IMPORTED) + set_target_properties(zlib PROPERTIES IMPORTED_LOCATION ${ZLIB_STATIC_LIB}) + + set(TRANSITIVE_LINK_LIBS + snappy + zlib + brotli_enc + brotli_dec + brotli_common + ) + set(ARROW_LINK_LIBS - arrow_static) + arrow_static + ${TRANSITIVE_LINK_LIBS}) endif() ############################################################# http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/benchmarks/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 61c1491..43af746 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -16,13 +16,14 @@ # under the License. SET(LINK_LIBS + ${ARROW_LINK_LIBS} thriftstatic) if (PARQUET_BUILD_BENCHMARKS) add_executable(decode_benchmark decode_benchmark.cc) # This uses private APIs - target_link_libraries(decode_benchmark ${LINK_LIBS} - arrow + target_link_libraries(decode_benchmark + ${LINK_LIBS} parquet_static) endif() http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/ci/travis_script_static.sh ---------------------------------------------------------------------- diff --git a/ci/travis_script_static.sh b/ci/travis_script_static.sh new file mode 100755 index 0000000..de57acc --- /dev/null +++ b/ci/travis_script_static.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +set -xe + +# Use a C++11 compiler on Linux +export CC="gcc-4.9" +export CXX="g++-4.9" + +# ---------------------------------------------------------------------- +# Set up external toolchain + +MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"; +export MINICONDA=$HOME/miniconda +wget -O miniconda.sh $MINICONDA_URL +bash miniconda.sh -b -p $MINICONDA +export PATH="$MINICONDA/bin:$PATH" +export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain + +conda update -y -q conda +conda config --set auto_update_conda false +conda info -a + +conda config --set show_channel_urls True + +# Help with SSL timeouts to S3 +conda config --set remote_connect_timeout_secs 12 + +conda info -a + +conda create -y -q -p $CPP_TOOLCHAIN \ + boost-cpp thrift-cpp cmake git \ + -c conda-forge + +# ---------------------------------------------------------------------- + +: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/parquet-build} +export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/data +export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN +export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH +export BOOST_ROOT=$CPP_TOOLCHAIN +export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/data +export SNAPPY_STATIC_LIB=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/snappy_ep/src/snappy_ep-install/lib/libsnappy.a +export BROTLI_STATIC_LIB_ENC=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/brotli_ep/src/brotli_ep-install/lib/x86_64-linux-gnu/libbrotlienc.a +export BROTLI_STATIC_LIB_DEC=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/brotli_ep/src/brotli_ep-install/lib/x86_64-linux-gnu/libbrotlidec.a +export BROTLI_STATIC_LIB_COMMON=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/brotli_ep/src/brotli_ep-install/lib/x86_64-linux-gnu/libbrotlicommon.a +export ZLIB_STATIC_LIB=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/zlib_ep/src/zlib_ep-install/lib/libz.a + + +cmake -DPARQUET_CXXFLAGS=-Werror \ + -DPARQUET_TEST_MEMCHECK=ON \ + -DPARQUET_ARROW_LINKAGE="static" \ + -DPARQUET_BUILD_SHARED=OFF \ + -DPARQUET_BOOST_USE_SHARED=OFF \ + -DPARQUET_BUILD_BENCHMARKS=ON \ + -DPARQUET_BUILD_EXAMPLES=ON \ + -DPARQUET_GENERATE_COVERAGE=1 \ + $TRAVIS_BUILD_DIR + +pushd $CPP_BUILD_DIR + +make -j4 || exit 1 +ctest -VV -L unittest || { cat $TRAVIS_BUILD_DIR/parquet-build/Testing/Temporary/LastTest.log; exit 1; } + +popd http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/cmake_modules/ThirdpartyToolchain.cmake ---------------------------------------------------------------------- diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 716debc..85630a4 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -332,6 +332,7 @@ if (NOT ARROW_FOUND) -DCMAKE_INSTALL_LIBDIR=${ARROW_LIB_DIR} -DARROW_JEMALLOC=OFF -DARROW_IPC=OFF + -DARROW_BUILD_SHARED=${PARQUET_BUILD_SHARED} -DARROW_BOOST_USE_SHARED=${PARQUET_BOOST_USE_SHARED} -DARROW_BUILD_TESTS=OFF) http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/examples/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 594295d..721fa9a 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -17,5 +17,5 @@ if (PARQUET_BUILD_EXECUTABLES) add_executable(reader-writer reader-writer.cc) - target_link_libraries(reader-writer parquet_shared arrow) + target_link_libraries(reader-writer parquet_static) endif() http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/src/parquet/arrow/reader.cc ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/reader.cc b/src/parquet/arrow/reader.cc index dd1c9d2..addfb85 100644 --- a/src/parquet/arrow/reader.cc +++ b/src/parquet/arrow/reader.cc @@ -226,7 +226,7 @@ class ColumnReader::Impl { }; // Reader implementation for primitive arrays -class PrimitiveImpl : public ColumnReader::Impl { +class PARQUET_NO_EXPORT PrimitiveImpl : public ColumnReader::Impl { public: PrimitiveImpl(MemoryPool* pool, std::unique_ptr<FileColumnIterator> input) : pool_(pool), @@ -300,7 +300,7 @@ class PrimitiveImpl : public ColumnReader::Impl { }; // Reader implementation for struct array -class StructImpl : public ColumnReader::Impl { +class PARQUET_NO_EXPORT StructImpl : public ColumnReader::Impl { public: explicit StructImpl(const std::vector<std::shared_ptr<Impl>>& children, int16_t struct_def_level, MemoryPool* pool, const NodePtr& node)
