PARQUET-416: C++11 compilation, code reorg, libparquet and installation targets
Reorganize code into a top level src/parquet directly, add a libparquet shared library, and add install targets for libparquet and its header files. Add cpplint script and `make lint` target for code linting. Replaces earlier PR #13 Author: Wes McKinney <[email protected]> Closes #14 from wesm/libparquet-library and squashes the following commits: 2e356fd [Wes McKinney] PARQUET-416: Compile with C++11 and replace usages of boost::shared_ptr with std::shared_ptr and other C++11 fixes. Reorganize code into a top level src/parquet directly, add a libparquet shared library, and add install targets for libparquet and its header files. Add cpplint script and `make lint` target for code linting. Project: http://git-wip-us.apache.org/repos/asf/parquet-cpp/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-cpp/commit/337cf584 Tree: http://git-wip-us.apache.org/repos/asf/parquet-cpp/tree/337cf584 Diff: http://git-wip-us.apache.org/repos/asf/parquet-cpp/diff/337cf584 Branch: refs/heads/master Commit: 337cf584ea25a1c6397c4412a706c58e4bd5e58c Parents: ea30dec Author: Wes McKinney <[email protected]> Authored: Fri Jan 8 15:49:25 2016 -0800 Committer: Nong Li <[email protected]> Committed: Fri Jan 8 15:49:25 2016 -0800 ---------------------------------------------------------------------- .travis.yml | 43 +- CMakeLists.txt | 127 +- build-support/cpplint.py | 6323 ++++++++++++++++++ cmake_modules/clean-all.cmake | 14 + example/CMakeLists.txt | 9 +- example/decode_benchmark.cc | 7 +- example/parquet_reader.cc | 26 +- generated/gen-cpp/CMakeLists.txt | 20 - generated/gen-cpp/parquet_constants.cpp | 17 - generated/gen-cpp/parquet_constants.h | 24 - generated/gen-cpp/parquet_types.cpp | 2006 ------ generated/gen-cpp/parquet_types.h | 1123 ---- src/CMakeLists.txt | 19 - src/compression/CMakeLists.txt | 18 - src/compression/codec.h | 72 - src/compression/lz4-codec.cc | 38 - src/compression/snappy-codec.cc | 40 - src/encodings/bool-encoding.h | 47 - src/encodings/delta-bit-pack-encoding.h | 114 - src/encodings/delta-byte-array-encoding.h | 73 - .../delta-length-byte-array-encoding.h | 62 - src/encodings/dictionary-encoding.h | 146 - src/encodings/encodings.h | 83 - src/encodings/plain-encoding.h | 82 - src/impala/bit-stream-utils.h | 145 - src/impala/bit-stream-utils.inline.h | 164 - src/impala/bit-util.h | 174 - src/impala/compiler-util.h | 38 - src/impala/logging.h | 31 - src/impala/rle-encoding.h | 417 -- src/parquet.cc | 35 +- src/parquet/CMakeLists.txt | 18 + src/parquet/compression/CMakeLists.txt | 30 + src/parquet/compression/codec.h | 71 + src/parquet/compression/lz4-codec.cc | 40 + src/parquet/compression/snappy-codec.cc | 42 + src/parquet/encodings/CMakeLists.txt | 24 + src/parquet/encodings/bool-encoding.h | 48 + src/parquet/encodings/delta-bit-pack-encoding.h | 116 + .../encodings/delta-byte-array-encoding.h | 74 + .../delta-length-byte-array-encoding.h | 63 + src/parquet/encodings/dictionary-encoding.h | 148 + src/parquet/encodings/encodings.h | 82 + src/parquet/encodings/plain-encoding.h | 83 + src/parquet/parquet.h | 38 +- src/parquet/thrift/CMakeLists.txt | 29 + src/parquet/thrift/parquet_constants.cpp | 17 + src/parquet/thrift/parquet_constants.h | 24 + src/parquet/thrift/parquet_types.cpp | 2006 ++++++ src/parquet/thrift/parquet_types.h | 1123 ++++ src/parquet/util/CMakeLists.txt | 24 + src/parquet/util/bit-stream-utils.h | 147 + src/parquet/util/bit-stream-utils.inline.h | 164 + src/parquet/util/bit-util.h | 174 + src/parquet/util/compiler-util.h | 37 + src/parquet/util/logging.h | 31 + src/parquet/util/rle-encoding.h | 419 ++ src/parquet/util/stopwatch.h | 49 + src/util/stopwatch.h | 49 - 59 files changed, 11630 insertions(+), 5077 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/337cf584/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 58b7641..5da9a6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ +sudo: required +dist: trusty language: cpp compiler: @@ -9,38 +11,51 @@ os: - osx addons: - apt: - packages: - - libboost-dev - #- libsnappy-dev currently handled by thirdparty scipts. - - libboost-program-options-dev #needed for thrift cpp compilation - - libboost-test-dev #needed for thrift cpp compilation - - libssl-dev #needed for thrift cpp compilation - - libtool #needed for thrift cpp compilation - - bison #needed for thrift cpp compilation - - flex #needed for thrift cpp compilation - - pkg-config #needed for thrift cpp compilation + apt: + sources: + - ubuntu-toolchain-r-test + - kalakris-cmake + packages: + - gcc-4.9 + - g++-4.9 + - cmake + - valgrind + - libboost-dev + #- libsnappy-dev currently handled by thirdparty scipts. + - libboost-program-options-dev #needed for thrift cpp compilation + - libboost-test-dev #needed for thrift cpp compilation + - libssl-dev #needed for thrift cpp compilation + - libtool #needed for thrift cpp compilation + - bison #needed for thrift cpp compilation + - flex #needed for thrift cpp compilation + - pkg-config #needed for thrift cpp compilation before_install: - pushd thirdparty # thrift cpp - > + if [ $TRAVIS_OS_NAME == osx ]; then + brew update && + brew install thrift; + fi + - > if [ $TRAVIS_OS_NAME == linux ]; then - wget http://www.us.apache.org/dist/thrift/0.9.1/thrift-0.9.1.tar.gz && + wget http://archive.apache.org/dist/thrift/0.9.1/thrift-0.9.1.tar.gz && tar xfz thrift-0.9.1.tar.gz && pushd thrift-0.9.1 && - ./configure --without-qt4 --without-c_glib --without-csharp --without-java --without-erlang --without-nodejs --without-lua --without-python --without-perl --without-php --without-php_extension --without-ruby --without-haskell --without-go --without-d --with-cpp --prefix=$HOME/local && + ./configure CXXFLAGS='-fPIC' --without-qt4 --without-c_glib --without-csharp --without-java --without-erlang --without-nodejs --without-lua --without-python --without-perl --without-php --without-php_extension --without-ruby --without-haskell --without-go --without-d --with-cpp --prefix=$HOME/local && make clean && make install && popd; fi - - if [ $TRAVIS_OS_NAME == osx ]; then brew install thrift; fi # snappy and lz4 - ./download_thirdparty.sh - ./build_thirdparty.sh - popd before_script: + - export CC="gcc-4.9" + - export CXX="g++-4.9" - mkdir build - cd build - THRIFT_HOME=$HOME/local cmake .. http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/337cf584/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index ad0ed5f..eb67f75 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,18 +12,64 @@ # See the License for the specific language governing permissions and # limitations under the License. - cmake_minimum_required(VERSION 2.6) +project(parquet-cpp) # generate CTest input files enable_testing() # where to find cmake modules set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules") +set(BUILD_SUPPORT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/build-support) set(THIRDPARTY_PREFIX ${CMAKE_SOURCE_DIR}/thirdparty/installed) set(CMAKE_PREFIX_PATH ${THIRDPARTY_PREFIX}) +if(APPLE) + set(CMAKE_MACOSX_RPATH 1) + set(CMAKE_OSX_DEPLOYMENT_TARGET 10.9) +endif() + +if (NOT PARQUET_LINK) + set(PARQUET_LINK "a") +elseif(NOT ("auto" MATCHES "^${PARQUET_LINK}" OR + "dynamic" MATCHES "^${PARQUET_LINK}" OR + "static" MATCHES "^${PARQUET_LINK}")) + message(FATAL_ERROR "Unknown value for PARQUET_LINK, must be auto|dynamic|static") +else() + # Remove all but the first letter. + string(SUBSTRING "${PARQUET_LINK}" 0 1 PARQUET_LINK) +endif() + +# if no build build type is specified, default to debug builds +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Debug) +endif(NOT CMAKE_BUILD_TYPE) + +# set compile output directory +string (TOLOWER ${CMAKE_BUILD_TYPE} BUILD_SUBDIR_NAME) + +# If build in-source, create the latest symlink. If build out-of-source, which is +# preferred, simply output the binaries in the build folder +if (${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR}) + set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/build/${BUILD_SUBDIR_NAME}/") + # Link build/latest to the current build directory, to avoid developers + # accidentally running the latest debug build when in fact they're building + # release builds. + FILE(MAKE_DIRECTORY ${BUILD_OUTPUT_ROOT_DIRECTORY}) + if (NOT APPLE) + set(MORE_ARGS "-T") + endif() +EXECUTE_PROCESS(COMMAND ln ${MORE_ARGS} -sf ${BUILD_OUTPUT_ROOT_DIRECTORY} + ${CMAKE_CURRENT_BINARY_DIR}/build/latest) +else() + set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}/") +endif() + +############################################################ +# Dependencies +############################################################ + # find boost headers and libs set(Boost_DEBUG TRUE) set(Boost_USE_MULTITHREADED ON) @@ -58,22 +104,83 @@ include_directories(SYSTEM ${LZ4_INCLUDE_DIR}) add_library(lz4static STATIC IMPORTED) set_target_properties(lz4static PROPERTIES IMPORTED_LOCATION ${LZ4_STATIC_LIB}) -SET(CMAKE_CXX_FLAGS "-msse4.2 -Wall -Wno-unused-value -Wno-unused-variable -Wno-sign-compare -Wno-unknown-pragmas") -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -ggdb") - # Thrift requires these definitions for some types that we use add_definitions(-DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H -DHAVE_NETDB_H) add_definitions(-fPIC) -# where to put generated libraries -set(LIBRARY_OUTPUT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/build") +# where to put generated archives (.a files) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") +set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") + +# where to put generated libraries (.so files) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") +set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") # where to put generated binaries -set(EXECUTABLE_OUTPUT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/bin") +set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}") + +SET(CMAKE_CXX_FLAGS "-std=c++11 -msse4.2 -Wall -Wno-unused-value -Wno-unused-variable -Wno-sign-compare -Wno-unknown-pragmas") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -ggdb") include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/generated) -add_subdirectory(generated/gen-cpp) -add_subdirectory(src) +############################################################ +# "make lint" target +############################################################ +if (UNIX) + # Full lint + add_custom_target(lint ${BUILD_SUPPORT_DIR}/cpplint.py + --verbose=4 + --filter=-whitespace/comments,-readability/todo,-build/header_guard,-build/include_order + `find ${CMAKE_CURRENT_SOURCE_DIR}/src -name \\*.cc -or -name \\*.h | sed -e '/parquet\\/thrift/g'`) +endif (UNIX) + +############################################################ +# Library config + +set(LIBPARQUET_SRCS + src/parquet.cc +) + +set(LIBPARQUET_LINK_LIBS + parquet_compression + parquet_thrift + thriftstatic +) + +if ("${PARQUET_LINK}" STREQUAL "d" OR "${PARQUET_LINK}" STREQUAL "a") + set(LIBPARQUET_LINKAGE "SHARED") +else() + set(LIBPARQUET_LINKAGE "STATIC") +endif() + +add_library(parquet + ${LIBPARQUET_LINKAGE} + ${LIBPARQUET_SRCS} +) +set_target_properties(parquet + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") +target_link_libraries(parquet ${LIBPARQUET_LINK_LIBS}) + +if(APPLE) + set_target_properties(parquet PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") +endif() + +add_subdirectory(src/parquet) +add_subdirectory(src/parquet/compression) +add_subdirectory(src/parquet/encodings) +add_subdirectory(src/parquet/thrift) +add_subdirectory(src/parquet/util) + add_subdirectory(example) + +add_custom_target(clean-all + COMMAND ${CMAKE_BUILD_TOOL} clean + COMMAND ${CMAKE_COMMAND} -P cmake_modules/clean-all.cmake +) + +# installation + +install(TARGETS parquet + LIBRARY DESTINATION lib)
