IMPALA-4669: [KRPC] Add kudu_rpc library to build Import FindKRPC.cmake from Apache Kudu.
Add some files to protoc-gen-krpc link to allow it to find symbols now defined within Impala (without linking all of Impala's libraries). Change-Id: I33203e95dff07c87a6ec5c7a31b7a583b91849bc Reviewed-on: http://gerrit.cloudera.org:8080/5719 Reviewed-by: Henry Robinson <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/11352619 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/11352619 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/11352619 Branch: refs/heads/master Commit: 113526198051f6810c84df513d507074856f5e4c Parents: 48539a1 Author: Henry Robinson <[email protected]> Authored: Tue Nov 8 13:55:19 2016 -0800 Committer: Impala Public Jenkins <[email protected]> Committed: Fri Aug 25 22:51:41 2017 +0000 ---------------------------------------------------------------------- CMakeLists.txt | 4 ++ be/CMakeLists.txt | 8 +++ be/src/common/global-flags.cc | 9 +++ be/src/common/init.cc | 5 +- be/src/common/logging.cc | 9 +-- be/src/kudu/rpc/CMakeLists.txt | 13 +++- cmake_modules/FindKRPC.cmake | 117 ++++++++++++++++++++++++++++++++++++ 7 files changed, 151 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index 31a3fc4..954193e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -320,6 +320,10 @@ if (NOT APPLE) endif() ################################################################### + +# Required for KRPC_GENERATE, which converts protobuf to stubs. +find_package(KRPC REQUIRED) + # KuduClient can use GLOG add_definitions(-DKUDU_HEADERS_USE_GLOG) if(NOT $ENV{KUDU_CLIENT_DIR} EQUAL "") http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/be/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 3ac4193..7bf9e0c 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -349,9 +349,14 @@ set (IMPALA_LINK_LIBS Exec Exprs GlobalFlags + histogram_proto ImpalaThrift kudu_util + krpc Rpc + rpc_header_proto + rpc_introspection_proto + pb_util_proto Runtime Scheduling security @@ -401,10 +406,12 @@ set (IMPALA_DEPENDENCIES ThriftSaslTransport openssl_ssl openssl_crypto + crcutil gutil glog gflags krb5 + libev pprof breakpad_client hdfs @@ -512,6 +519,7 @@ add_subdirectory(src/common) add_subdirectory(src/exec) add_subdirectory(src/exprs) add_subdirectory(src/kudu/security) +add_subdirectory(src/kudu/rpc) add_subdirectory(src/kudu/util) add_subdirectory(src/runtime) add_subdirectory(src/scheduling) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/be/src/common/global-flags.cc ---------------------------------------------------------------------- diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc index 5d61edc..9404148 100644 --- a/be/src/common/global-flags.cc +++ b/be/src/common/global-flags.cc @@ -161,3 +161,12 @@ DEFINE_int64(inc_stats_size_limit_bytes, 200 * (1LL<<20), "Maximum size of " DEFINE_bool(enable_stats_extrapolation, false, "If true, uses table statistics computed with COMPUTE STATS " "to extrapolate the row counts of partitions."); + +DEFINE_string(log_filename, "", + "Prefix of log filename - " + "full path is <log_dir>/<log_filename>.[INFO|WARN|ERROR|FATAL]"); +DEFINE_bool(redirect_stdout_stderr, true, + "If true, redirects stdout/stderr to INFO/ERROR log."); +DEFINE_int32(max_log_files, 10, "Maximum number of log files to retain per severity " + "level. The most recent log files are retained. If set to 0, all log files are " + "retained."); http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/be/src/common/init.cc ---------------------------------------------------------------------- diff --git a/be/src/common/init.cc b/be/src/common/init.cc index 9734610..6b48bb8 100644 --- a/be/src/common/init.cc +++ b/be/src/common/init.cc @@ -62,10 +62,7 @@ DECLARE_string(hostname); DECLARE_int32(logbufsecs); DECLARE_int32(max_minidumps); DECLARE_string(redaction_rules_file); - -DEFINE_int32(max_log_files, 10, "Maximum number of log files to retain per severity " - "level. The most recent log files are retained. If set to 0, all log files are " - "retained."); +DECLARE_int32(max_log_files); DEFINE_int32(max_audit_event_log_files, 0, "Maximum number of audit event log files " "to retain. The most recent audit event log files are retained. If set to 0, " http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/be/src/common/logging.cc ---------------------------------------------------------------------- diff --git a/be/src/common/logging.cc b/be/src/common/logging.cc index 8025c9f..021af4a 100644 --- a/be/src/common/logging.cc +++ b/be/src/common/logging.cc @@ -40,14 +40,9 @@ #include "common/names.h" -DEFINE_string(log_filename, "", - "Prefix of log filename - " - "full path is <log_dir>/<log_filename>.[INFO|WARN|ERROR|FATAL]"); -DEFINE_bool(redirect_stdout_stderr, true, - "If true, redirects stdout/stderr to INFO/ERROR log."); - DECLARE_string(redaction_rules_file); - +DECLARE_string(log_filename); +DECLARE_bool(redirect_stdout_stderr); DECLARE_string(audit_event_log_dir); using boost::uuids::random_generator; http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/be/src/kudu/rpc/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/be/src/kudu/rpc/CMakeLists.txt b/be/src/kudu/rpc/CMakeLists.txt index 0cfe6e9..643b854 100644 --- a/be/src/kudu/rpc/CMakeLists.txt +++ b/be/src/kudu/rpc/CMakeLists.txt @@ -86,14 +86,21 @@ ADD_EXPORTABLE_LIBRARY(krpc DEPS ${KRPC_LIBS}) ### RPC generator tool -add_executable(protoc-gen-krpc protoc-gen-krpc.cc) +add_executable(protoc-gen-krpc protoc-gen-krpc.cc + # Impala - add stub for kudu::VersionInfo + ${CMAKE_CURRENT_SOURCE_DIR}/../../common/kudu_version.cc + # Impala - add definition for any flag names shared between Impala / Kudu. + # TODO: Consider either removing code that depends on these flags, or namespacing them + # somehow. + ${CMAKE_CURRENT_SOURCE_DIR}/../../common/global-flags.cc) target_link_libraries(protoc-gen-krpc - ${KUDU_BASE_LIBS} + ${KUDU_BASE_LIBS} rpc_header_proto protoc protobuf gutil - kudu_util) + kudu_util + ) #### RPC test PROTOBUF_GENERATE_CPP( http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/11352619/cmake_modules/FindKRPC.cmake ---------------------------------------------------------------------- diff --git a/cmake_modules/FindKRPC.cmake b/cmake_modules/FindKRPC.cmake new file mode 100644 index 0000000..593edcd --- /dev/null +++ b/cmake_modules/FindKRPC.cmake @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Kudu RPC generator support +######### +find_package(Protobuf REQUIRED) + +# +# Generate the KRPC files for a given .proto file. +# +function(KRPC_GENERATE SRCS HDRS TGTS) + if(NOT ARGN) + message(SEND_ERROR "Error: KRPC_GENERATE() called without protobuf files") + return() + endif(NOT ARGN) + + set(options) + set(one_value_args SOURCE_ROOT BINARY_ROOT) + set(multi_value_args EXTRA_PROTO_PATHS PROTO_FILES) + cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + set(${SRCS}) + set(${HDRS}) + set(${TGTS}) + + set(EXTRA_PROTO_PATH_ARGS) + foreach(PP ${ARG_EXTRA_PROTO_PATHS}) + set(EXTRA_PROTO_PATH_ARGS ${EXTRA_PROTO_PATH_ARGS} --proto_path ${PP}) + endforeach() + + if("${ARG_SOURCE_ROOT}" STREQUAL "") + SET(ARG_SOURCE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") + endif() + GET_FILENAME_COMPONENT(ARG_SOURCE_ROOT ${ARG_SOURCE_ROOT} ABSOLUTE) + + if("${ARG_BINARY_ROOT}" STREQUAL "") + SET(ARG_BINARY_ROOT "${CMAKE_CURRENT_BINARY_DIR}") + endif() + GET_FILENAME_COMPONENT(ARG_BINARY_ROOT ${ARG_BINARY_ROOT} ABSOLUTE) + + foreach(FIL ${ARG_PROTO_FILES}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + + # Ensure that the protobuf file is within the source root. + # This is a requirement of protoc. + FILE(RELATIVE_PATH PROTO_REL_TO_ROOT "${ARG_SOURCE_ROOT}" "${ABS_FIL}") + + GET_FILENAME_COMPONENT(REL_DIR "${PROTO_REL_TO_ROOT}" PATH) + + if(NOT REL_DIR STREQUAL "") + SET(REL_DIR "${REL_DIR}/") + endif() + + set(PROTO_CC_OUT "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.pb.cc") + set(PROTO_H_OUT "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.pb.h") + set(SERVICE_CC "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.service.cc") + set(SERVICE_H "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.service.h") + set(PROXY_CC "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.proxy.cc") + set(PROXY_H "${ARG_BINARY_ROOT}/${REL_DIR}${FIL_WE}.proxy.h") + list(APPEND ${SRCS} "${PROTO_CC_OUT}" "${SERVICE_CC}" "${PROXY_CC}") + list(APPEND ${HDRS} "${PROTO_H_OUT}" "${SERVICE_H}" "${PROXY_H}") + + get_filename_component(PROTO_LIB_DIR ${PROTOBUF_SHARED_LIBRARY} DIRECTORY) + + add_custom_command( + OUTPUT "${SERVICE_CC}" "${SERVICE_H}" "${PROXY_CC}" "${PROXY_H}" + "${PROTO_CC_OUT}" "${PROTO_H_OUT}" + COMMAND ${CMAKE_COMMAND} + -E env "LD_LIBRARY_PATH=${PROTO_LIB_DIR}:$ENV{LD_LIBRARY_PATH}" + ${PROTOBUF_PROTOC_EXECUTABLE} + ARGS --plugin $<TARGET_FILE:protoc-gen-krpc> + --plugin $<TARGET_FILE:protoc-gen-insertions> + --cpp_out ${ARG_BINARY_ROOT} + --krpc_out ${ARG_BINARY_ROOT} + --insertions_out ${ARG_BINARY_ROOT} + --proto_path ${ARG_SOURCE_ROOT} + # Used to find built-in .proto files (e.g. FileDescriptorProto) + --proto_path ${PROTOBUF_INCLUDE_DIR} + ${EXTRA_PROTO_PATH_ARGS} ${ABS_FIL} + DEPENDS ${ABS_FIL} protoc-gen-krpc protoc-gen-insertions + COMMENT "Running C++ protocol buffer compiler with KRPC plugin on ${FIL}" + VERBATIM) + + # This custom target enforces that there's just one invocation of protoc + # when there are multiple consumers of the generated files. The target name + # must be unique; adding parts of the filename helps ensure this. + set(TGT_NAME ${REL_DIR}${FIL}) + string(REPLACE "/" "-" TGT_NAME ${TGT_NAME}) + add_custom_target(${TGT_NAME} + DEPENDS "${SERVICE_CC}" "${SERVICE_H}" + "${PROXY_CC}" "${PROXY_H}" + "${PROTO_CC_OUT}" "${PROTO_H_OUT}") + list(APPEND ${TGTS} "${TGT_NAME}") + endforeach() + + set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) + set(${SRCS} ${${SRCS}} PARENT_SCOPE) + set(${HDRS} ${${HDRS}} PARENT_SCOPE) + set(${TGTS} ${${TGTS}} PARENT_SCOPE) +endfunction()
