Repository: parquet-cpp Updated Branches: refs/heads/master e6763722b -> a5e6482e2
PARQUET-679: [C++] Resolve unit tests issues on Windows; Run unit tes⦠â¦ts with Appveyor; Appveyor build jobs for Visual Studio cmake generator and Debug configuration. Author: Max Risuhin <[email protected]> Closes #319 from MaxRis/PARQUET-679 and squashes the following commits: 6bf2cd7 [Max Risuhin] PARQUET-679: [C++] Resolve unit tests issues on Windows; Run unit tests with Appveyor; Appveyor build jobs for Visual Studio cmake generator and Debug configuration. Project: http://git-wip-us.apache.org/repos/asf/parquet-cpp/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-cpp/commit/a5e6482e Tree: http://git-wip-us.apache.org/repos/asf/parquet-cpp/tree/a5e6482e Diff: http://git-wip-us.apache.org/repos/asf/parquet-cpp/diff/a5e6482e Branch: refs/heads/master Commit: a5e6482e2909b89022374c4bddb70fb1b5eb63cf Parents: e676372 Author: Max Risuhin <[email protected]> Authored: Tue May 9 19:15:24 2017 +0200 Committer: Uwe L. Korn <[email protected]> Committed: Tue May 9 19:15:24 2017 +0200 ---------------------------------------------------------------------- CMakeLists.txt | 14 +++++ appveyor.yml | 10 +++- ci/msvc-build.bat | 12 +++- cmake_modules/ThirdpartyToolchain.cmake | 69 ++++++++++++++-------- src/parquet/arrow/arrow-reader-writer-test.cc | 8 +-- src/parquet/util/bit-util.h | 2 +- 6 files changed, 81 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/a5e6482e/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index 46d4394..e18cf7b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -153,6 +153,17 @@ set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") # where to put generated binaries set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}") +if (MSVC) + # define output directories to overwrite default Visual Studio generator output directories + FOREACH(BUILD_CONFIG ${CMAKE_CONFIGURATION_TYPES}) + string(TOUPPER ${BUILD_CONFIG} UPPERCASE_BUILD_CONFIG) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY}) + set(CMAKE_PDB_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY}) + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY}) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY}) + ENDFOREACH() +endif() + ############################################################ # Benchmarking ############################################################ @@ -353,6 +364,9 @@ endif() # Debug symbols are stripped for reduced binary size. Add # -DPARQUET_CXXFLAGS="-g" to include them set(CXX_FLAGS_DEBUG "-ggdb -O0") +if (MSVC) + set(CXX_FLAGS_DEBUG "${CXX_FLAGS_DEBUG} /bigobj") +endif() set(CXX_FLAGS_FASTDEBUG "-ggdb -O1") set(CXX_FLAGS_RELEASE "-O3") http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/a5e6482e/appveyor.yml ---------------------------------------------------------------------- diff --git a/appveyor.yml b/appveyor.yml index 437cd6d..f3fac11 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,14 +23,22 @@ environment: - GENERATOR: NMake Makefiles PYTHON: "3.5" ARCH: "64" + + - GENERATOR: Visual Studio 14 2015 Win64 + PYTHON: "3.5" + ARCH: "64" MSVC_DEFAULT_OPTIONS: ON BOOST_ROOT: C:\Libraries\boost_1_63_0 BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0 +configuration: + - Debug + - Release + init: - set MINICONDA=C:\Miniconda35-x64 - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH% - if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 build_script: - - call ci\msvc-build.bat \ No newline at end of file + - call ci\msvc-build.bat http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/a5e6482e/ci/msvc-build.bat ---------------------------------------------------------------------- diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat index 7e723fa..b1b54b4 100644 --- a/ci/msvc-build.bat +++ b/ci/msvc-build.bat @@ -20,8 +20,16 @@ mkdir build cd build +SET PARQUET_TEST_DATA=%APPVEYOR_BUILD_FOLDER%\data + cmake -G "%GENERATOR%" ^ - -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ + -DPARQUET_BOOST_USE_SHARED=OFF ^ + -DPARQUET_CXXFLAGS="/MP" ^ .. || exit /B -nmake || exit /B \ No newline at end of file +cmake --build . --config %CONFIGURATION% || exit /B + +if "%CONFIGURATION%" == "Release" ( + ctest -VV || exit /B +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/a5e6482e/cmake_modules/ThirdpartyToolchain.cmake ---------------------------------------------------------------------- diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 44b2619..74b4632 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -82,7 +82,7 @@ if (PARQUET_BOOST_USE_SHARED) endif() find_package(Boost COMPONENTS regex REQUIRED) - if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG") set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG}) else() set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE}) @@ -91,7 +91,7 @@ else() # Find static Boost libraries. set(Boost_USE_STATIC_LIBS ON) find_package(Boost COMPONENTS regex REQUIRED) - if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG") set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG}) else() set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE}) @@ -129,7 +129,11 @@ if (NOT ZLIB_FOUND) set(ZLIB_HOME "${ZLIB_PREFIX}") set(ZLIB_INCLUDE_DIRS "${ZLIB_PREFIX}/include") if (MSVC) - set(ZLIB_STATIC_LIB_NAME zlibstatic.lib) + if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") + set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib) + else() + set(ZLIB_STATIC_LIB_NAME zlibstatic.lib) + endif() else() set(ZLIB_STATIC_LIB_NAME libz.a) endif() @@ -175,7 +179,11 @@ if (NOT THRIFT_FOUND) set(THRIFT_HOME "${THRIFT_PREFIX}") set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include") IF (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/libthriftd.a") + IF (MSVC) + set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/thriftmdd.lib") + ELSE() + set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/libthriftd.a") + ENDIF() ELSE() IF (MSVC) set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/thriftmd.lib") @@ -206,7 +214,9 @@ if (NOT THRIFT_FOUND) "-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIRS}" "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}" "-DWITH_SHARED_LIB=OFF" + "-DWITH_PLUGIN=OFF" ${THRIFT_CMAKE_ARGS}) + set (THRIFT_DEPENDENCIES zlib_ep) endif() if (CMAKE_VERSION VERSION_GREATER "3.2") @@ -215,12 +225,14 @@ if (NOT THRIFT_FOUND) URL "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz"; BUILD_BYPRODUCTS "${THRIFT_STATIC_LIB}" "${THRIFT_COMPILER}" CMAKE_ARGS ${THRIFT_CMAKE_ARGS} - STEP_TARGETS flex_step libevent_step) + STEP_TARGETS flex_step libevent_step + DEPENDS ${THRIFT_DEPENDENCIES}) else() ExternalProject_Add(thrift_ep URL "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz"; CMAKE_ARGS ${THRIFT_CMAKE_ARGS} - STEP_TARGETS flex_step libevent_step) + STEP_TARGETS flex_step libevent_step + DEPENDS ${THRIFT_DEPENDENCIES}) endif() set(THRIFT_VENDORED 1) else() @@ -235,27 +247,28 @@ if (MSVC) # Download and configure Windows build of Flex and Bison ExternalProject_Add_Step(thrift_ep flex_step - COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/dist/winflexbison && - cd thirdparty/dist/winflexbison && - curl -SLO https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip && - ${CMAKE_COMMAND} -E tar xzf win_flex_bison-${WINFLEXBISON_VERSION}.zip + COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/dist/winflexbison + COMMAND cd thirdparty/dist/winflexbison + COMMAND curl -SLO https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip + COMMAND ${CMAKE_COMMAND} -E tar xzf win_flex_bison-${WINFLEXBISON_VERSION}.zip DEPENDERS configure + DEPENDEES download WORKING_DIRECTORY ${SOURCE_DIR}) # Download and build libevent ExternalProject_Add_Step(thrift_ep libevent_step - COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/src && - cd thirdparty/src && - curl -SLO https://github.com/nmathewson/Libevent/archive/release-${LIBEVENT_VERSION}-rc.zip && - ${CMAKE_COMMAND} -E tar xzf release-${LIBEVENT_VERSION}-rc.zip && - cd Libevent-release-${LIBEVENT_VERSION}-rc && - nmake -f Makefile.nmake && - ${CMAKE_COMMAND} -E make_directory lib && - copy *.lib lib && - xcopy /E /I /D WIN32-Code\\nmake include\\event2 && - xcopy /E /I /D WIN32-Code\\nmake\\event2 include\\event2 && - copy *.h include + COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/src + COMMAND cd thirdparty/src + COMMAND curl -SLO https://github.com/nmathewson/Libevent/archive/release-${LIBEVENT_VERSION}-rc.zip + COMMAND ${CMAKE_COMMAND} -E tar xzf release-${LIBEVENT_VERSION}-rc.zip + COMMAND cd Libevent-release-${LIBEVENT_VERSION}-rc + COMMAND ${CMAKE_COMMAND} -E make_directory build + COMMAND cd build + COMMAND ${CMAKE_COMMAND} -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=.. .. + COMMAND nmake + COMMAND nmake install DEPENDERS configure + DEPENDEES download WORKING_DIRECTORY ${SOURCE_DIR}) endif() @@ -554,13 +567,16 @@ if (NOT ARROW_FOUND) -DARROW_IPC=OFF -DARROW_BOOST_USE_SHARED=${PARQUET_BOOST_USE_SHARED} -DARROW_BUILD_TESTS=OFF) - + if (MSVC) - set(ARROW_CMAKE_ARGS -G "NMake Makefiles" ${ARROW_CMAKE_ARGS}) + set(ARROW_CMAKE_ARGS -G "NMake Makefiles" -DARROW_CXXFLAGS="/WX" ${ARROW_CMAKE_ARGS}) + if (NOT PARQUET_BOOST_USE_SHARED) + set(ARROW_CMAKE_ARGS -DARROW_BOOST_USE_SHARED=OFF ${ARROW_CMAKE_ARGS}) + endif() endif() if ("$ENV{PARQUET_ARROW_VERSION}" STREQUAL "") - set(ARROW_VERSION "f7ab7270bb07466dabf84c015a6db2a192eb3dad") + set(ARROW_VERSION "2c3e111d45c056d429cef312533c9f3f96b08ae8") else() set(ARROW_VERSION "$ENV{PARQUET_ARROW_VERSION}") endif() @@ -581,7 +597,7 @@ if (NOT ARROW_FOUND) CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp CMAKE_GENERATOR "NMake Makefiles" CMAKE_GENERATOR_PLATFORM "x64" - BUILD_COMMAND nmake && nmake install + BUILD_COMMAND nmake COMMAND nmake install STEP_TARGETS copy_dll_step) else() ExternalProject_Add(arrow_ep @@ -589,12 +605,13 @@ if (NOT ARROW_FOUND) CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp CMAKE_GENERATOR "NMake Makefiles" CMAKE_GENERATOR_PLATFORM "x64" - BUILD_COMMAND nmake && nmake install + BUILD_COMMAND nmake COMMAND nmake install STEP_TARGETS copy_dll_step) endif() ExternalProject_Get_Property(arrow_ep SOURCE_DIR) ExternalProject_Add_Step(arrow_ep copy_dll_step + COMMAND ${CMAKE_COMMAND} -E make_directory ${BUILD_OUTPUT_ROOT_DIRECTORY} COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_SHARED_LIB} ${BUILD_OUTPUT_ROOT_DIRECTORY} DEPENDEES build WORKING_DIRECTORY ${SOURCE_DIR}) http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/a5e6482e/src/parquet/arrow/arrow-reader-writer-test.cc ---------------------------------------------------------------------- diff --git a/src/parquet/arrow/arrow-reader-writer-test.cc b/src/parquet/arrow/arrow-reader-writer-test.cc index 48790d9..64690ae 100644 --- a/src/parquet/arrow/arrow-reader-writer-test.cc +++ b/src/parquet/arrow/arrow-reader-writer-test.cc @@ -661,10 +661,10 @@ TEST_F(TestInt96ParquetIO, ReadIntoTimestamp) { // 2nd January 1970, 11:35min 145738543ns Int96 day; - day.value[2] = 2440589l; + day.value[2] = UINT32_C(2440589); int64_t seconds = (11 * 60 + 35) * 60; *(reinterpret_cast<int64_t*>(&(day.value))) = - seconds * 1000l * 1000l * 1000l + 145738543; + seconds * INT64_C(1000) * INT64_C(1000) * INT64_C(1000) + 145738543; // Compute the corresponding nanosecond timestamp struct tm datetime = {0}; datetime.tm_year = 70; @@ -676,7 +676,7 @@ TEST_F(TestInt96ParquetIO, ReadIntoTimestamp) { epoch.tm_year = 70; epoch.tm_mday = 1; // Nanoseconds since the epoch - int64_t val = lrint(difftime(mktime(&datetime), mktime(&epoch))) * 1000000000; + int64_t val = lrint(difftime(mktime(&datetime), mktime(&epoch))) * INT64_C(1000000000); val += 145738543; std::vector<std::shared_ptr<schema::Node>> fields( @@ -1064,7 +1064,7 @@ TEST(TestArrowReadWrite, ReadColumnSubset) { TEST(TestArrowWrite, CheckChunkSize) { const int num_columns = 2; const int num_rows = 128; - const int64_t chunk_size = 0; // note the chunk_size is 0 + const int64_t chunk_size = 0; // note the chunk_size is 0 std::shared_ptr<Table> table; MakeDoubleTable(num_columns, num_rows, 1, &table); http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/a5e6482e/src/parquet/util/bit-util.h ---------------------------------------------------------------------- diff --git a/src/parquet/util/bit-util.h b/src/parquet/util/bit-util.h index 8f0a270..559d5d5 100644 --- a/src/parquet/util/bit-util.h +++ b/src/parquet/util/bit-util.h @@ -329,7 +329,7 @@ class BitUtil { // Returns the minimum number of bits needed to represent the value of 'x' static inline int NumRequiredBits(uint64_t x) { for (int i = 63; i >= 0; --i) { - if (x & 1L << i) return i + 1; + if (x & (UINT64_C(1) << i)) return i + 1; } return 0; }
