IMPALA-4650: Add Protobuf to build This patch adds Protobuf 2.6.1 to Impala's build, and bumps the toolchain version so that the dependency is available. Protobuf is unused in this commit, but is required for KRPC.
FindProtobuf.cmake includes some utility CMake methods to generate source code from Protobuf definitions. It is taken from Kudu. Change-Id: Ic9357fe0f201cbf7df1ba19fe4773dfb6c10b4ef Reviewed-on: http://gerrit.cloudera.org:8080/5657 Tested-by: Impala Public Jenkins Reviewed-by: Henry Robinson <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/4b3fdc33 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/4b3fdc33 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/4b3fdc33 Branch: refs/heads/master Commit: 4b3fdc33019ffd6eeee075e9c8f233d99f066121 Parents: 5d028d9 Author: Henry Robinson <[email protected]> Authored: Tue Oct 25 14:53:12 2016 -0700 Committer: Henry Robinson <[email protected]> Committed: Thu Jan 12 05:18:17 2017 +0000 ---------------------------------------------------------------------- CMakeLists.txt | 12 ++ be/CMakeLists.txt | 1 + bin/bootstrap_toolchain.py | 3 +- bin/impala-config.sh | 3 +- cmake_modules/FindProtobuf.cmake | 233 ++++++++++++++++++++++++++++++++++ 5 files changed, 250 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b3fdc33/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c5d0d3..18be5d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -81,6 +81,7 @@ set_dep_root(LLVM) set(LLVM_DEBUG_ROOT $ENV{IMPALA_TOOLCHAIN}/llvm-$ENV{IMPALA_LLVM_DEBUG_VERSION}) set_dep_root(LZ4) set_dep_root(OPENLDAP) +set_dep_root(PROTOBUF) set_dep_root(RE2) set_dep_root(RAPIDJSON) set_dep_root(SNAPPY) @@ -298,6 +299,17 @@ if (NOT APPLE) SHARED_LIB "${DL_LIB_PATH}") endif() +# find protobuf headers, libs and compiler +find_package(Protobuf REQUIRED) +include_directories(SYSTEM ${PROTOBUF_INCLUDE_DIR}) +message(STATUS "Protobuf include dir: " ${PROTOBUF_INCLUDE_DIR}) + +ADD_THIRDPARTY_LIB(protobuf + STATIC_LIB "${PROTOBUF_STATIC_LIBRARY}") +ADD_THIRDPARTY_LIB(protoc + STATIC_LIB "${PROTOBUF_PROTOC_STATIC_LIBRARY}" + DEPS protobuf) + ################################################################### # KuduClient can use GLOG http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b3fdc33/be/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 6a076f8..3697ceb 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -239,6 +239,7 @@ set(CLANG_INCLUDE_FLAGS "-I${ZLIB_INCLUDE_DIR}" "-I${OPENSSL_INCLUDE_DIR}" "-I${LDAP_INCLUDE_DIR}" + "-I${PROTOBUF_INCLUDE_DIR}" ) # allow linking of static libs into dynamic lib http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b3fdc33/bin/bootstrap_toolchain.py ---------------------------------------------------------------------- diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py index dbe53a2..c8e35de 100755 --- a/bin/bootstrap_toolchain.py +++ b/bin/bootstrap_toolchain.py @@ -351,7 +351,8 @@ if __name__ == "__main__": packages = ["avro", "binutils", "boost", "breakpad", "bzip2", "cmake", "gcc", "gflags", "glog", "gperftools", "gtest", "kudu", "llvm", ("llvm", "3.8.0-asserts-p1"), "lz4", - "openldap", "rapidjson", "re2", "snappy", "thrift", "tpc-h", "tpc-ds", "zlib"] + "openldap", "protobuf", "rapidjson", "re2", "snappy", "thrift", "tpc-h", "tpc-ds", + "zlib"] bootstrap(toolchain_root, packages) # Download the CDH components if necessary. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b3fdc33/bin/impala-config.sh ---------------------------------------------------------------------- diff --git a/bin/impala-config.sh b/bin/impala-config.sh index f185861..c980ea3 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -72,7 +72,7 @@ fi # moving to a different build of the toolchain, e.g. when a version is bumped or a # compile option is changed. The build id can be found in the output of the toolchain # build jobs, it is constructed from the build number and toolchain git hash prefix. -export IMPALA_TOOLCHAIN_BUILD_ID=308-96a4cc516e +export IMPALA_TOOLCHAIN_BUILD_ID=333-f7c19a394a # Versions of toolchain dependencies. # ----------------------------------- @@ -96,6 +96,7 @@ export IMPALA_LLVM_DEBUG_VERSION=3.8.0-asserts-p1 export IMPALA_LZ4_VERSION=svn export IMPALA_OPENLDAP_VERSION=2.4.25 export IMPALA_OPENSSL_VERSION=0.9.8zf +export IMPALA_PROTOBUF_VERSION=2.6.1 export IMPALA_POSTGRES_JDBC_DRIVER_VERSION=9.0-801 export IMPALA_RAPIDJSON_VERSION=0.11 export IMPALA_RE2_VERSION=20130115-p1 http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b3fdc33/cmake_modules/FindProtobuf.cmake ---------------------------------------------------------------------- diff --git a/cmake_modules/FindProtobuf.cmake b/cmake_modules/FindProtobuf.cmake new file mode 100644 index 0000000..a1092c1 --- /dev/null +++ b/cmake_modules/FindProtobuf.cmake @@ -0,0 +1,233 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#============================================================================= +# This file is heavily modified/rewritten from FindProtobuf.cmake from the +# CMake project: +# +# Copyright 2011 Kirill A. Korinskiy <[email protected]> +# Copyright 2009 Kitware, Inc. +# Copyright 2009 Philip Lowman <[email protected]> +# Copyright 2008 Esben Mose Hansen, Ange Optimization ApS +# +# Distributed under the OSI-approved BSD License (the "License"): +# +# CMake - Cross Platform Makefile Generator +# Copyright 2000-2015 Kitware, Inc. +# Copyright 2000-2011 Insight Software Consortium +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the names of Kitware, Inc., the Insight Software Consortium, +# nor the names of their contributors may be used to endorse or promote +# products derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#============================================================================= + +######### +# Local rewrite of the protobuf support in cmake. +# +# Supports cross-module protobuf dependencies and protobufs inside +# packages much better than the one built into cmake. +######### +# +# Locate and configure the Google Protocol Buffers library. +# Defines the following variables: +# +# PROTOBUF_INCLUDE_DIR - the include directory for protocol buffers +# PROTOBUF_SHARED_LIBRARY - path to protobuf's shared library +# PROTOBUF_STATIC_LIBRARY - path to protobuf's static library +# PROTOBUF_PROTOC_SHARED_LIBRARY - path to protoc's shared library +# PROTOBUF_PROTOC_STATIC_LIBRARY - path to protoc's static library +# PROTOBUF_PROTOC_EXECUTABLE - the protoc compiler +# PROTOBUF_FOUND - whether the Protocol Buffers library has been found +# +# ==================================================================== +# Example: +# +# find_package(Protobuf REQUIRED) +# include_directories(${PROTOBUF_INCLUDE_DIR}) +# +# include_directories(${CMAKE_CURRENT_BINARY_DIR}) +# PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS PROTO_TGTS +# [SOURCE_ROOT <root from which source is found>] +# [BINARY_ROOT <root into which binaries are built>] +# PROTO_FILES foo.proto) +# add_executable(bar bar.cc ${PROTO_SRCS} ${PROTO_HDRS}) +# target_link_libraries(bar ${PROTOBUF_SHARED_LIBRARY}) +# +# NOTE: You may need to link against pthreads, depending +# on the platform. +# ==================================================================== +# +# PROTOBUF_GENERATE_CPP (public function) +# SRCS = Variable to define with autogenerated +# source files +# HDRS = Variable to define with autogenerated +# header files +# TGTS = Variable to define with autogenerated +# custom targets; if SRCS/HDRS need to be used in multiple +# libraries, those libraries should depend on these targets +# in order to "serialize" the protoc invocations +# ==================================================================== + +function(PROTOBUF_GENERATE_CPP SRCS HDRS TGTS) + if(NOT ARGN) + message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files") + return() + endif(NOT ARGN) + + set(options) + set(one_value_args SOURCE_ROOT BINARY_ROOT) + set(multi_value_args EXTRA_PROTO_PATHS PROTO_FILES) + cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + + set(${SRCS}) + set(${HDRS}) + set(${TGTS}) + + set(EXTRA_PROTO_PATH_ARGS) + foreach(PP ${ARG_EXTRA_PROTO_PATHS}) + set(EXTRA_PROTO_PATH_ARGS ${EXTRA_PROTO_PATH_ARGS} --proto_path ${PP}) + endforeach() + + if("${ARG_SOURCE_ROOT}" STREQUAL "") + SET(ARG_SOURCE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") + endif() + GET_FILENAME_COMPONENT(ARG_SOURCE_ROOT ${ARG_SOURCE_ROOT} ABSOLUTE) + + if("${ARG_BINARY_ROOT}" STREQUAL "") + SET(ARG_BINARY_ROOT "${CMAKE_CURRENT_BINARY_DIR}") + endif() + GET_FILENAME_COMPONENT(ARG_BINARY_ROOT ${ARG_BINARY_ROOT} ABSOLUTE) + + foreach(FIL ${ARG_PROTO_FILES}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + + # Ensure that the protobuf file is within the source root. + # This is a requirement of protoc. + FILE(RELATIVE_PATH PROTO_REL_TO_ROOT "${ARG_SOURCE_ROOT}" "${ABS_FIL}") + + GET_FILENAME_COMPONENT(REL_DIR "${PROTO_REL_TO_ROOT}" PATH) + + if(NOT REL_DIR STREQUAL "") + SET(REL_DIR "${REL_DIR}/") + endif() + + set(PROTO_CC_OUT "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.pb.cc") + set(PROTO_H_OUT "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.pb.h") + list(APPEND ${SRCS} "${PROTO_CC_OUT}") + list(APPEND ${HDRS} "${PROTO_H_OUT}") + + add_custom_command( + OUTPUT "${PROTO_CC_OUT}" "${PROTO_H_OUT}" + COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + ARGS + --plugin $<TARGET_FILE:protoc-gen-insertions> + --cpp_out ${ARG_BINARY_ROOT} + --insertions_out ${ARG_BINARY_ROOT} + --proto_path ${ARG_SOURCE_ROOT} + # Used to find built-in .proto files (e.g. FileDescriptorProto) + --proto_path ${PROTOBUF_INCLUDE_DIR} + ${EXTRA_PROTO_PATH_ARGS} ${ABS_FIL} + DEPENDS ${ABS_FIL} protoc-gen-insertions + COMMENT "Running C++ protocol buffer compiler on ${FIL}" + VERBATIM ) + + # This custom target enforces that there's just one invocation of protoc + # when there are multiple consumers of the generated files. The target name + # must be unique; adding parts of the filename helps ensure this. + set(TGT_NAME ${REL_DIR}${FIL}) + string(REPLACE "/" "-" TGT_NAME ${TGT_NAME}) + add_custom_target(${TGT_NAME} + DEPENDS "${PROTO_CC_OUT}" "${PROTO_H_OUT}") + list(APPEND ${TGTS} "${TGT_NAME}") + endforeach() + + set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) + set(${SRCS} ${${SRCS}} PARENT_SCOPE) + set(${HDRS} ${${HDRS}} PARENT_SCOPE) + set(${TGTS} ${${TGTS}} PARENT_SCOPE) +endfunction() + + +find_path(PROTOBUF_INCLUDE_DIR google/protobuf/service.h + PATHS ${PROTOBUF_ROOT}/include + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) + +find_library(PROTOBUF_SHARED_LIBRARY protobuf + PATHS ${PROTOBUF_ROOT}/lib + DOC "The Google Protocol Buffers Library" + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) + +find_library(PROTOBUF_STATIC_LIBRARY libprotobuf.a + PATHS ${PROTOBUF_ROOT}/lib + DOC "Static version of the Google Protocol Buffers Library" + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) + +find_library(PROTOBUF_PROTOC_SHARED_LIBRARY protoc + PATHS ${PROTOBUF_ROOT}/lib + DOC "The Google Protocol Buffers Compiler Library" + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) + +find_library(PROTOBUF_PROTOC_STATIC_LIBRARY libprotoc.a + PATHS ${PROTOBUF_ROOT}/lib + DOC "Static version of the Google Protocol Buffers Compiler Library" + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) + +find_program(PROTOBUF_PROTOC_EXECUTABLE protoc + PATHS ${PROTOBUF_ROOT}/bin + DOC "The Google Protocol Buffers Compiler" + NO_CMAKE_SYSTEM_PATH + NO_SYSTEM_ENVIRONMENT_PATH) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(PROTOBUF REQUIRED_VARS + PROTOBUF_SHARED_LIBRARY PROTOBUF_STATIC_LIBRARY + PROTOBUF_PROTOC_SHARED_LIBRARY PROTOBUF_PROTOC_STATIC_LIBRARY + PROTOBUF_INCLUDE_DIR PROTOBUF_PROTOC_EXECUTABLE)
