This is an automated email from the ASF dual-hosted git repository.
fsaintjacques pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 0fc5bc4 ARROW-5710: [C++] Allow compiling Gandiva with Ninja on
Windows
0fc5bc4 is described below
commit 0fc5bc429fbe527b1e42db4307cde8d0ce2818c6
Author: Antoine Pitrou <anto...@python.org>
AuthorDate: Tue Jun 25 14:28:33 2019 -0400
ARROW-5710: [C++] Allow compiling Gandiva with Ninja on Windows
Ninja is faster than msbuild, especially in combination with clcache.
Author: Antoine Pitrou <anto...@python.org>
Closes #4679 from pitrou/ARROW-5710 and squashes the following commits:
b9a40210c <Antoine Pitrou> ARROW-5710: Allow compiling Gandiva with Ninja
on Windows
---
appveyor.yml | 2 +-
ci/cpp-msvc-build-main.bat | 14 +++++++--
cpp/src/gandiva/engine.cc | 5 +--
cpp/src/gandiva/make_precompiled_bitcode.py | 49 +++++++++++++++++++++++++++++
cpp/src/gandiva/precompiled/CMakeLists.txt | 21 ++++---------
cpp/src/gandiva/precompiled_bitcode.cc.in | 2 +-
cpp/src/gandiva/tests/CMakeLists.txt | 15 ++++-----
python/manylinux1/build_arrow.sh | 5 +--
8 files changed, 82 insertions(+), 31 deletions(-)
diff --git a/appveyor.yml b/appveyor.yml
index c5eadf6..e84d78c 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -65,7 +65,7 @@ environment:
CONFIGURATION: "Release"
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
- JOB: "Toolchain"
- GENERATOR: Visual Studio 14 2015 Win64
+ GENERATOR: Ninja
CONFIGURATION: "Release"
ARROW_BUILD_FLIGHT: "ON"
ARROW_BUILD_GANDIVA: "ON"
diff --git a/ci/cpp-msvc-build-main.bat b/ci/cpp-msvc-build-main.bat
index 80a4cca..4ab4136 100644
--- a/ci/cpp-msvc-build-main.bat
+++ b/ci/cpp-msvc-build-main.bat
@@ -31,6 +31,14 @@ if "%JOB%" == "Toolchain" (
set CMAKE_ARGS=%CMAKE_ARGS% -DARROW_DEPENDENCY_SOURCE=AUTO
)
+if "%USE_CLCACHE%" == "true" (
+ @rem XXX Without forcing CMAKE_CXX_COMPILER, CMake can re-run itself and
+ @rem unfortunately switch from Release to Debug mode...
+ set CMAKE_ARGS=%CMAKE_ARGS% ^
+ -DARROW_USE_CLCACHE=ON ^
+ -DCMAKE_CXX_COMPILER=clcache
+)
+
@rem Retrieve git submodules, configure env var for Parquet unit tests
git submodule update --init || exit /B
set ARROW_TEST_DATA=%CD%\testing\data
@@ -51,14 +59,14 @@ mkdir cpp\build
pushd cpp\build
cmake -G "%GENERATOR%" %CMAKE_ARGS% ^
- -DCMAKE_VERBOSE_MAKEFILE=OFF ^
+ -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^
+ -DCMAKE_VERBOSE_MAKEFILE=OFF ^
+ -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
-DARROW_BOOST_USE_SHARED=OFF ^
- -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_BUILD_STATIC=OFF ^
-DARROW_BUILD_TESTS=ON ^
-DARROW_BUILD_EXAMPLES=ON ^
- -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^
-DARROW_CXXFLAGS="%ARROW_CXXFLAGS%" ^
-DCMAKE_CXX_FLAGS_RELEASE="/MD %CMAKE_CXX_FLAGS_RELEASE%" ^
-DARROW_FLIGHT=%ARROW_BUILD_FLIGHT% ^
diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc
index 5abe4b5..e6112b3 100644
--- a/cpp/src/gandiva/engine.cc
+++ b/cpp/src/gandiva/engine.cc
@@ -66,7 +66,7 @@
namespace gandiva {
-extern const char kPrecompiledBitcode[];
+extern const unsigned char kPrecompiledBitcode[];
extern const size_t kPrecompiledBitcodeSize;
std::once_flag init_once_flag;
@@ -130,7 +130,8 @@ Status Engine::Make(std::shared_ptr<Configuration> config,
// Handling for pre-compiled IR libraries.
Status Engine::LoadPreCompiledIR() {
- auto bitcode = llvm::StringRef(kPrecompiledBitcode, kPrecompiledBitcodeSize);
+ auto bitcode = llvm::StringRef(reinterpret_cast<const
char*>(kPrecompiledBitcode),
+ kPrecompiledBitcodeSize);
/// Read from file into memory buffer.
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> buffer_or_error =
diff --git a/cpp/src/gandiva/make_precompiled_bitcode.py
b/cpp/src/gandiva/make_precompiled_bitcode.py
new file mode 100644
index 0000000..97d96f8
--- /dev/null
+++ b/cpp/src/gandiva/make_precompiled_bitcode.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+
+marker = b"<DATA_CHARS>"
+
+def expand(data):
+ """
+ Expand *data* as a initializer list of hexadecimal char escapes.
+ """
+ expanded_data = ", ".join([hex(c) for c in bytearray(data)])
+ return expanded_data.encode('ascii')
+
+
+def apply_template(template, data):
+ if template.count(marker) != 1:
+ raise ValueError("Invalid template")
+ return template.replace(marker, expand(data))
+
+
+if __name__ == "__main__":
+ if len(sys.argv) != 4:
+ raise ValueError("Usage: {0} <template file> <data file> "
+ "<output file>".format(sys.argv[0]))
+ with open(sys.argv[1], "rb") as f:
+ template = f.read()
+ with open(sys.argv[2], "rb") as f:
+ data = f.read()
+
+ expanded_data = apply_template(template, data)
+ with open(sys.argv[3], "wb") as f:
+ f.write(expanded_data)
diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt
b/cpp/src/gandiva/precompiled/CMakeLists.txt
index ea089d2..0202fc5 100644
--- a/cpp/src/gandiva/precompiled/CMakeLists.txt
+++ b/cpp/src/gandiva/precompiled/CMakeLists.txt
@@ -78,22 +78,13 @@ add_custom_command(OUTPUT ${GANDIVA_PRECOMPILED_BC_PATH}
${BC_FILES}
DEPENDS ${BC_FILES})
-# write a cmake script to replace precompiled bitcode file's content into a
.cc file
-file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/WritePrecompiledCC.cmake" "\
- file(READ \"${GANDIVA_PRECOMPILED_BC_PATH}\" HEXCONTENT HEX)
- string(REGEX REPLACE
- \"([0-9a-f][0-9a-f])\"
- \"'\\\\\\\\x\\\\1',\"
- GANDIVA_PRECOMPILED_BITCODE_CHARS
- \"\${HEXCONTENT}\")
- configure_file(${GANDIVA_PRECOMPILED_CC_IN_PATH}
- ${GANDIVA_PRECOMPILED_CC_PATH})
-")
-
-# add the previous command to the execution chain
+# turn the bitcode file into a C++ static data variable.
add_custom_command(OUTPUT ${GANDIVA_PRECOMPILED_CC_PATH}
- COMMAND ${CMAKE_COMMAND} -P
-
"${CMAKE_CURRENT_BINARY_DIR}/WritePrecompiledCC.cmake"
+ COMMAND python
+
"${CMAKE_CURRENT_SOURCE_DIR}/../make_precompiled_bitcode.py"
+ ${GANDIVA_PRECOMPILED_CC_IN_PATH}
+ ${GANDIVA_PRECOMPILED_BC_PATH}
+ ${GANDIVA_PRECOMPILED_CC_PATH}
DEPENDS ${GANDIVA_PRECOMPILED_CC_IN_PATH}
${GANDIVA_PRECOMPILED_BC_PATH})
diff --git a/cpp/src/gandiva/precompiled_bitcode.cc.in
b/cpp/src/gandiva/precompiled_bitcode.cc.in
index 1d5985a..9c38296 100644
--- a/cpp/src/gandiva/precompiled_bitcode.cc.in
+++ b/cpp/src/gandiva/precompiled_bitcode.cc.in
@@ -20,7 +20,7 @@
namespace gandiva {
// Content of precompiled bitcode file.
-extern const char kPrecompiledBitcode[] = {
${GANDIVA_PRECOMPILED_BITCODE_CHARS} };
+extern const unsigned char kPrecompiledBitcode[] = { <DATA_CHARS> };
extern const size_t kPrecompiledBitcodeSize = sizeof(kPrecompiledBitcode);
} // namespace gandiva
diff --git a/cpp/src/gandiva/tests/CMakeLists.txt
b/cpp/src/gandiva/tests/CMakeLists.txt
index 2235269..5fa2da1 100644
--- a/cpp/src/gandiva/tests/CMakeLists.txt
+++ b/cpp/src/gandiva/tests/CMakeLists.txt
@@ -32,10 +32,11 @@ add_gandiva_test(decimal_test)
add_gandiva_test(decimal_single_test)
add_gandiva_test(filter_project_test)
-add_gandiva_test(projector_test_static SOURCES projector_test.cc
USE_STATIC_LINKING)
-
-add_arrow_benchmark(micro_benchmarks
- PREFIX
- "gandiva"
- EXTRA_LINK_LIBS
- gandiva_static)
+if(ARROW_BUILD_STATIC)
+ add_gandiva_test(projector_test_static SOURCES projector_test.cc
USE_STATIC_LINKING)
+ add_arrow_benchmark(micro_benchmarks
+ PREFIX
+ "gandiva"
+ EXTRA_LINK_LIBS
+ gandiva_static)
+endif()
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 867d6ff..ca5ab7c 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -54,7 +54,8 @@ mkdir -p /io/dist
CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})"
PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python"
PIP="${CPYTHON_PATH}/bin/pip"
-PATH="${PATH}:${CPYTHON_PATH}"
+# Put our Python first to avoid picking up an antiquated Python from CMake
+PATH="${CPYTHON_PATH}/bin:${PATH}"
if [ "${PYTHON_VERSION}" != "2.7" ]; then
export PYARROW_WITH_FLIGHT=1
@@ -76,7 +77,7 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries
==="
ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}-${UNICODE_WIDTH}
mkdir -p "${ARROW_BUILD_DIR}"
pushd "${ARROW_BUILD_DIR}"
-PATH="${CPYTHON_PATH}/bin:${PATH}" cmake -DCMAKE_BUILD_TYPE=Release \
+cmake -DCMAKE_BUILD_TYPE=Release \
-DARROW_DEPENDENCY_SOURCE="SYSTEM" \
-DZLIB_ROOT=/usr/local \
-DCMAKE_INSTALL_PREFIX=/arrow-dist \
