This is an automated email from the ASF dual-hosted git repository. ravindra pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push: new ba9f809 ARROW-4959:[C++][Gandiva][Crossbow] Gandiva crossbow packaging changes. ba9f809 is described below commit ba9f809e3c7c194e5c46bb1ff57c51ef3049aa7a Author: Praveen <prav...@dremio.com> AuthorDate: Thu Apr 4 14:23:05 2019 +0530 ARROW-4959:[C++][Gandiva][Crossbow] Gandiva crossbow packaging changes. - Supports building Gandiva in the manylinux docker image. - Fixes some issues discovered. - Move crossbow to build using manylinux docker image. - Use system dependencies to package Gandiva in OsX. Author: Praveen <prav...@dremio.com> Closes #4082 from praveenbingo/ARROW-4959-CentOS5 and squashes the following commits: e5913de2 <Praveen> Fix review comments. d5c6e5c3 <Praveen> Address review comments. ab73e1ee <Praveen> Fix format issues. 936f05a7 <Praveen> Fix ci issues. f0de7f63 <Praveen> Remove unnecessary files. 7627e282 <Praveen> Fix path for osx. 9cfc2023 <Praveen> Copy dylib. fba3b4a8 <Praveen> Add a dummy target. bb02b5ee <Praveen> Support both Java7 and Java8 environments. b6b4a9ad <Praveen> Try OSX fixes. aa13c3bb <Praveen> Fix java path. a92841bf <Praveen> Fix path. 5e9c86b8 <Praveen> Fix paths. 5447d115 <Praveen> Misc fixes. 5880d1ca <Praveen> ARROW-4959: Package gandiva in manylinux docker. --- cpp/src/gandiva/jni/CMakeLists.txt | 1 + cpp/src/gandiva/jni/expression_registry_helper.cc | 4 +- .../gandiva/precompiled/extended_math_ops_test.cc | 56 +++++++++++-------- dev/tasks/gandiva-jars/README.md | 29 ++++++++++ dev/tasks/gandiva-jars/build-cpp-linux.sh | 64 ++++++++++++++++++++++ .../{build-cpp.sh => build-cpp-osx.sh} | 6 +- dev/tasks/gandiva-jars/build-java.sh | 8 +-- dev/tasks/gandiva-jars/travis.linux.yml | 10 ++-- dev/tasks/gandiva-jars/travis.osx.yml | 9 ++- java/gandiva/CMakeLists.txt | 46 +++++++++++++--- .../arrow/gandiva/evaluator/ProjectorTest.java | 3 +- 11 files changed, 185 insertions(+), 51 deletions(-) diff --git a/cpp/src/gandiva/jni/CMakeLists.txt b/cpp/src/gandiva/jni/CMakeLists.txt index af58efd..680b6e5 100644 --- a/cpp/src/gandiva/jni/CMakeLists.txt +++ b/cpp/src/gandiva/jni/CMakeLists.txt @@ -74,6 +74,7 @@ add_arrow_lib(gandiva_jni DEPENDENCIES gandiva_java gandiva_jni_proto + gandiva_jni_headers EXTRA_INCLUDES $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/include> diff --git a/cpp/src/gandiva/jni/expression_registry_helper.cc b/cpp/src/gandiva/jni/expression_registry_helper.cc index 94c4833..2275641 100644 --- a/cpp/src/gandiva/jni/expression_registry_helper.cc +++ b/cpp/src/gandiva/jni/expression_registry_helper.cc @@ -150,7 +150,7 @@ Java_org_apache_arrow_gandiva_evaluator_ExpressionRegistryJniHelper_getGandivaSu types::ExtGandivaType* gandiva_data_type = gandiva_data_types.add_datatype(); ArrowToProtobuf(type, gandiva_data_type); } - int size = static_cast<int>(gandiva_data_types.ByteSizeLong()); + int size = gandiva_data_types.ByteSize(); std::unique_ptr<jbyte[]> buffer{new jbyte[size]}; gandiva_data_types.SerializeToArray(reinterpret_cast<void*>(buffer.get()), size); jbyteArray ret = env->NewByteArray(size); @@ -179,7 +179,7 @@ Java_org_apache_arrow_gandiva_evaluator_ExpressionRegistryJniHelper_getGandivaSu ArrowToProtobuf(param_type, proto_param_type); } } - int size = static_cast<int>(gandiva_functions.ByteSizeLong()); + int size = gandiva_functions.ByteSize(); std::unique_ptr<jbyte[]> buffer{new jbyte[size]}; gandiva_functions.SerializeToArray(reinterpret_cast<void*>(buffer.get()), size); jbyteArray ret = env->NewByteArray(size); diff --git a/cpp/src/gandiva/precompiled/extended_math_ops_test.cc b/cpp/src/gandiva/precompiled/extended_math_ops_test.cc index fc9fadd..4f91260 100644 --- a/cpp/src/gandiva/precompiled/extended_math_ops_test.cc +++ b/cpp/src/gandiva/precompiled/extended_math_ops_test.cc @@ -15,67 +15,75 @@ // specific language governing permissions and limitations // under the License. +#include <math.h> + #include <gtest/gtest.h> #include "gandiva/execution_context.h" #include "gandiva/precompiled/types.h" namespace gandiva { +static const double MAX_ERROR = 0.00005; + +void VerifyFuzzyEquals(double actual, double expected, double max_error = MAX_ERROR) { + EXPECT_TRUE(fabs(actual - expected) < max_error) << actual << " != " << expected; +} + TEST(TestExtendedMathOps, TestCbrt) { - EXPECT_EQ(cbrt_int32(27), 3); - EXPECT_EQ(cbrt_int64(27), 3); - EXPECT_EQ(cbrt_float32(27), 3); - EXPECT_EQ(cbrt_float64(27), 3); - EXPECT_EQ(cbrt_float64(-27), -3); - - EXPECT_EQ(cbrt_float32(15.625), 2.5); - EXPECT_EQ(cbrt_float64(15.625), 2.5); + VerifyFuzzyEquals(cbrt_int32(27), 3); + VerifyFuzzyEquals(cbrt_int64(27), 3); + VerifyFuzzyEquals(cbrt_float32(27), 3); + VerifyFuzzyEquals(cbrt_float64(27), 3); + VerifyFuzzyEquals(cbrt_float64(-27), -3); + + VerifyFuzzyEquals(cbrt_float32(15.625), 2.5); + VerifyFuzzyEquals(cbrt_float64(15.625), 2.5); } TEST(TestExtendedMathOps, TestExp) { double val = 20.085536923187668; - EXPECT_EQ(exp_int32(3), val); - EXPECT_EQ(exp_int64(3), val); - EXPECT_EQ(exp_float32(3), val); - EXPECT_EQ(exp_float64(3), val); + VerifyFuzzyEquals(exp_int32(3), val); + VerifyFuzzyEquals(exp_int64(3), val); + VerifyFuzzyEquals(exp_float32(3), val); + VerifyFuzzyEquals(exp_float64(3), val); } TEST(TestExtendedMathOps, TestLog) { double val = 4.1588830833596715; - EXPECT_EQ(log_int32(64), val); - EXPECT_EQ(log_int64(64), val); - EXPECT_EQ(log_float32(64), val); - EXPECT_EQ(log_float64(64), val); + VerifyFuzzyEquals(log_int32(64), val); + VerifyFuzzyEquals(log_int64(64), val); + VerifyFuzzyEquals(log_float32(64), val); + VerifyFuzzyEquals(log_float64(64), val); EXPECT_EQ(log_int32(0), -std::numeric_limits<double>::infinity()); } TEST(TestExtendedMathOps, TestLog10) { - EXPECT_EQ(log10_int32(100), 2); - EXPECT_EQ(log10_int64(100), 2); - EXPECT_EQ(log10_float32(100), 2); - EXPECT_EQ(log10_float64(100), 2); + VerifyFuzzyEquals(log10_int32(100), 2); + VerifyFuzzyEquals(log10_int64(100), 2); + VerifyFuzzyEquals(log10_float32(100), 2); + VerifyFuzzyEquals(log10_float64(100), 2); } TEST(TestExtendedMathOps, TestPower) { - EXPECT_EQ(power_float64_float64(2, 5.4), 42.22425314473263); - EXPECT_EQ(power_float64_float64(5.4, 2), 29.160000000000004); + VerifyFuzzyEquals(power_float64_float64(2, 5.4), 42.22425314473263); + VerifyFuzzyEquals(power_float64_float64(5.4, 2), 29.160000000000004); } TEST(TestExtendedMathOps, TestLogWithBase) { gandiva::ExecutionContext context; float64 out = log_int32_int32(reinterpret_cast<int64>(&context), 1 /*base*/, 10 /*value*/); - EXPECT_EQ(out, 0); + VerifyFuzzyEquals(out, 0); EXPECT_EQ(context.has_error(), true); EXPECT_TRUE(context.get_error().find("divide by zero error") != std::string::npos) << context.get_error(); gandiva::ExecutionContext context1; out = log_int32_int32(reinterpret_cast<int64>(&context), 2 /*base*/, 64 /*value*/); - EXPECT_EQ(out, 6); + VerifyFuzzyEquals(out, 6); EXPECT_EQ(context1.has_error(), false); } diff --git a/dev/tasks/gandiva-jars/README.md b/dev/tasks/gandiva-jars/README.md new file mode 100644 index 0000000..2f4c694 --- /dev/null +++ b/dev/tasks/gandiva-jars/README.md @@ -0,0 +1,29 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +# Updating manylinux for Gandiva Jar Build. + +Do the following to update arrow manylinux docker image for building Gandiva Jars + +- Install java in the manylinux image. +- To do above, update Dockerfile-x86_64_base under python/manylinux1 to install java. +- Please note only upto java7 is available in CentOS5, so install java7 in the base. +- Export JAVA_HOME environment variable. +- Then update build_boost.sh under python/manylinux1/scripts to build boost statically. + +Please look at https://github.com/praveenbingo/arrow/tree/buildGandivaDocker that already has these changes. \ No newline at end of file diff --git a/dev/tasks/gandiva-jars/build-cpp-linux.sh b/dev/tasks/gandiva-jars/build-cpp-linux.sh new file mode 100755 index 0000000..e1df5fc --- /dev/null +++ b/dev/tasks/gandiva-jars/build-cpp-linux.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source /multibuild/manylinux_utils.sh + +# Quit on failure +set -e + +# Print commands for debugging +set -x + +ARROW_BUILD_DIR=/tmp/arrow-build +mkdir -p "${ARROW_BUILD_DIR}" +pushd "${ARROW_BUILD_DIR}" + +cmake -DCMAKE_BUILD_TYPE=Release \ + -DARROW_DEPENDENCY_SOURCE="SYSTEM" \ + -DZLIB_ROOT=/usr/local \ + -DCMAKE_INSTALL_PREFIX=/arrow-dist \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DARROW_BUILD_TESTS=ON \ + -DARROW_BUILD_SHARED=ON \ + -DARROW_BOOST_USE_SHARED=OFF \ + -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2;-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2/x86_64-CentOS-linux/" \ + -DARROW_JEMALLOC=ON \ + -DARROW_RPATH_ORIGIN=ON \ + -DARROW_PYTHON=OFF \ + -DARROW_PARQUET=OFF \ + -DARROW_GANDIVA=ON \ + -DARROW_GANDIVA_JAVA=ON \ + -DARROW_GANDIVA_JAVA7=ON \ + -DBoost_NAMESPACE=arrow_boost \ + -Ddouble-conversion_SOURCE=BUNDLED \ + -Dgflags_SOURCE=BUNDLED \ + -DRapidJSON_SOURCE=BUNDLED \ + -DRE2_SOURCE=BUNDLED \ + -DARROW_BUILD_UTILITIES=OFF \ + -DBoost_NAMESPACE=arrow_boost \ + -DBOOST_ROOT=/arrow_boost_dist \ + -GNinja /arrow/cpp +ninja install +ninja test +popd + + +# copy the library to distribution +cp -L /arrow-dist/lib/libgandiva_jni.so /io/dist + diff --git a/dev/tasks/gandiva-jars/build-cpp.sh b/dev/tasks/gandiva-jars/build-cpp-osx.sh similarity index 94% rename from dev/tasks/gandiva-jars/build-cpp.sh rename to dev/tasks/gandiva-jars/build-cpp-osx.sh index b585b9d..8272943 100755 --- a/dev/tasks/gandiva-jars/build-cpp.sh +++ b/dev/tasks/gandiva-jars/build-cpp-osx.sh @@ -19,10 +19,10 @@ set -e -source arrow/ci/travis_env_common.sh +set -x # Builds arrow + gandiva and tests the same. -pushd arrow/cpp +pushd cpp mkdir build pushd build CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release \ @@ -40,5 +40,7 @@ pushd arrow/cpp cmake $CMAKE_FLAGS .. make -j4 ctest + + cp -L release/libgandiva_jni.dylib $TRAVIS_BUILD_DIR/dist popd popd diff --git a/dev/tasks/gandiva-jars/build-java.sh b/dev/tasks/gandiva-jars/build-java.sh index d099035..bbbf74a 100755 --- a/dev/tasks/gandiva-jars/build-java.sh +++ b/dev/tasks/gandiva-jars/build-java.sh @@ -19,11 +19,9 @@ set -e -source arrow/ci/travis_env_common.sh +CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/dist/ -CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp/build/release - -pushd arrow/java +pushd java if [ $TRAVIS_OS_NAME == "linux" ]; then ldd $CPP_BUILD_DIR/libgandiva_jni.so fi @@ -33,5 +31,5 @@ pushd arrow/java # test only gandiva mvn test -P gandiva -pl gandiva -Dgandiva.cpp.build.dir=$CPP_BUILD_DIR # copy the jars to distribution folder - find gandiva/target/ -name "*.jar" -not -name "*tests*" -exec cp {} ../../dist/ \; + find gandiva/target/ -name "*.jar" -not -name "*tests*" -exec cp {} $CPP_BUILD_DIR \; popd diff --git a/dev/tasks/gandiva-jars/travis.linux.yml b/dev/tasks/gandiva-jars/travis.linux.yml index 7c5eb33..1e87e3e 100644 --- a/dev/tasks/gandiva-jars/travis.linux.yml +++ b/dev/tasks/gandiva-jars/travis.linux.yml @@ -44,14 +44,14 @@ before_script: - git clone -b {{ arrow.branch }} {{ arrow.remote }} arrow - git -C arrow checkout {{ arrow.head }} - export TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR/arrow - - arrow/ci/travis_install_linux.sh - - arrow/ci/travis_install_clang_tools.sh - - arrow/ci/travis_install_toolchain.sh + - script: + - cd $TRAVIS_BUILD_DIR - mkdir -p dist - - arrow/dev/tasks/gandiva-jars/build-cpp.sh || travis_terminate 1 - - arrow/dev/tasks/gandiva-jars/build-java.sh || travis_terminate 1 + # please refer to README for steps to update this image + - docker run -v $PWD:/io -v $PWD:/arrow quay.io/praveenbingo/arrow:buildGandivaDocker /io/dev/tasks/gandiva-jars/build-cpp-linux.sh || travis_terminate 1 + - dev/tasks/gandiva-jars/build-java.sh || travis_terminate 1 deploy: provider: releases diff --git a/dev/tasks/gandiva-jars/travis.osx.yml b/dev/tasks/gandiva-jars/travis.osx.yml index b801725..b3139f2 100644 --- a/dev/tasks/gandiva-jars/travis.osx.yml +++ b/dev/tasks/gandiva-jars/travis.osx.yml @@ -36,12 +36,15 @@ before_script: - git clone -b {{ arrow.branch }} {{ arrow.remote }} arrow - git -C arrow checkout {{ arrow.head }} - export TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR/arrow - - arrow/ci/travis_install_toolchain.sh + - brew update + - brew upgrade cmake + - brew install bison flex llvm@7 script: + - cd arrow - mkdir -p dist - - arrow/dev/tasks/gandiva-jars/build-cpp.sh || travis_terminate 1 - - arrow/dev/tasks/gandiva-jars/build-java.sh || travis_terminate 1 + - dev/tasks/gandiva-jars/build-cpp-osx.sh || travis_terminate 1 + - dev/tasks/gandiva-jars/build-java.sh || travis_terminate 1 deploy: provider: releases diff --git a/java/gandiva/CMakeLists.txt b/java/gandiva/CMakeLists.txt index 815acea..0a7c4d0 100644 --- a/java/gandiva/CMakeLists.txt +++ b/java/gandiva/CMakeLists.txt @@ -23,12 +23,40 @@ include(UseJava) include(FindJNI) message("generating headers to ${JNI_HEADERS_DIR}/jni") -add_jar(gandiva_java - src/main/java/org/apache/arrow/gandiva/evaluator/ConfigurationBuilder.java - src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java - src/main/java/org/apache/arrow/gandiva/evaluator/ExpressionRegistryJniHelper.java - src/main/java/org/apache/arrow/gandiva/exceptions/GandivaException.java - GENERATE_NATIVE_HEADERS - gandivajni_headers - DESTINATION - ${JNI_HEADERS_DIR}/jni) + +# generate_native_headers is available only from java8 +# centos5 does not have java8 images, so supporting java 7 too. +# unfortunately create_javah does not work in java8 correctly. +if(ARROW_GANDIVA_JAVA7) + add_jar( + gandiva_java + src/main/java/org/apache/arrow/gandiva/evaluator/ConfigurationBuilder.java + src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java + src/main/java/org/apache/arrow/gandiva/evaluator/ExpressionRegistryJniHelper.java + src/main/java/org/apache/arrow/gandiva/exceptions/GandivaException.java) + + create_javah(TARGET + gandiva_jni_headers + CLASSES + org.apache.arrow.gandiva.evaluator.ConfigurationBuilder + org.apache.arrow.gandiva.evaluator.JniWrapper + org.apache.arrow.gandiva.evaluator.ExpressionRegistryJniHelper + org.apache.arrow.gandiva.exceptions.GandivaException + DEPENDS + gandiva_java + CLASSPATH + gandiva_java + OUTPUT_DIR + ${JNI_HEADERS_DIR}/jni) +else() + add_jar( + gandiva_java + src/main/java/org/apache/arrow/gandiva/evaluator/ConfigurationBuilder.java + src/main/java/org/apache/arrow/gandiva/evaluator/JniWrapper.java + src/main/java/org/apache/arrow/gandiva/evaluator/ExpressionRegistryJniHelper.java + src/main/java/org/apache/arrow/gandiva/exceptions/GandivaException.java + GENERATE_NATIVE_HEADERS + gandiva_jni_headers + DESTINATION + ${JNI_HEADERS_DIR}/jni) +endif() diff --git a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java index d333556..e5f9fe6 100644 --- a/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java +++ b/java/gandiva/src/test/java/org/apache/arrow/gandiva/evaluator/ProjectorTest.java @@ -326,7 +326,7 @@ public class ProjectorTest extends BaseEvaluatorTest { AtomicInteger errorCount = new AtomicInteger(0); AtomicInteger errorCountExp = new AtomicInteger(0); // pre-build the projector so that same projector is used for all executions. - Projector.make(s, exprs); + Projector test = Projector.make(s, exprs); IntStream.range(0, 1000).forEach(i -> { executors.submit(() -> { @@ -373,6 +373,7 @@ public class ProjectorTest extends BaseEvaluatorTest { }); executors.shutdown(); executors.awaitTermination(100, java.util.concurrent.TimeUnit.SECONDS); + test.close(); Assert.assertEquals(errorCountExp.intValue(), errorCount.intValue()); }