IMPALA-3253: Modify gen_build_version.sh to always output the right version
gen_build_version.sh previously had a --noclean option which did not overwrite the version information if it was already populated. Since --noclean was the default option, it always never updated the version information. This patch modifies gen_build_version.py to generate a common/version.cc instead of a common/version.h. Now, common/version.h will be a part of the git repo and will not need to be modified on every build. It declares the functions that will return the build information. These functions will be defined in common/version.cc and the build information will change on every new build. Since only the .cc file changes on every build, we will not incur a highly noticable change in build times. Also changed the function names from GetImpalaBuild...() to GetImpaladBuild...() so as to avoid naming confusion between the Impala-lzo and the Impala functions. There is an accompanying change in the Impala-lzo library too. Change-Id: Ie461110b6f8ca545f04ea33b7b502aea550b8551 Reviewed-on: http://gerrit.cloudera.org:8080/2651 Reviewed-by: Sailesh Mukil <[email protected]> Tested-by: Sailesh Mukil <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/5d9e13a6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/5d9e13a6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/5d9e13a6 Branch: refs/heads/master Commit: 5d9e13a6af1e3da80d7361ca591704aa94e59099 Parents: 7cb8631 Author: Sailesh Mukil <[email protected]> Authored: Mon Mar 28 16:54:41 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Tue Jul 5 13:37:25 2016 -0700 ---------------------------------------------------------------------- be/.gitignore | 2 +- be/src/common/CMakeLists.txt | 12 +++++++ be/src/common/version.h | 22 ++++++++++++ be/src/exec/hdfs-lzo-text-scanner.cc | 12 +++---- be/src/exec/hdfs-parquet-table-writer.cc | 4 +-- be/src/service/impala-hs2-server.cc | 2 +- be/src/util/debug-util.cc | 6 ++-- bin/gen_build_version.py | 48 +++++++++++++-------------- bin/make_impala.sh | 2 +- 9 files changed, 72 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5d9e13a6/be/.gitignore ---------------------------------------------------------------------- diff --git a/be/.gitignore b/be/.gitignore index 9b20560..6c9804d 100644 --- a/be/.gitignore +++ b/be/.gitignore @@ -2,5 +2,5 @@ build generated-sources/**/* -src/common/version.h +src/common/version.cc http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5d9e13a6/be/src/common/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/be/src/common/CMakeLists.txt b/be/src/common/CMakeLists.txt index 00ba7f4..c9236bd 100644 --- a/be/src/common/CMakeLists.txt +++ b/be/src/common/CMakeLists.txt @@ -20,10 +20,22 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/common") # where to put generated binaries set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/common") +set(VERSION_CC_GEN_OUTPUT + ${CMAKE_SOURCE_DIR}/be/src/common/version.cc +) + add_library(Common init.cc logging.cc status.cc + ${VERSION_CC_GEN_OUTPUT} +) + +add_custom_command( + OUTPUT ${VERSION_CC_GEN_OUTPUT} + COMMAND $ENV{IMPALA_HOME}/bin/gen_build_version.py + COMMENT "Generating the version.cc file" + VERBATIM ) add_dependencies(Common thrift-deps) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5d9e13a6/be/src/common/version.h ---------------------------------------------------------------------- diff --git a/be/src/common/version.h b/be/src/common/version.h new file mode 100644 index 0000000..58183ca --- /dev/null +++ b/be/src/common/version.h @@ -0,0 +1,22 @@ +// Copyright 2016 Cloudera Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IMPALA_COMMON_VERSION_H +#define IMPALA_COMMON_VERSION_H + +const char* GetDaemonBuildVersion(); +const char* GetDaemonBuildHash(); +const char* GetDaemonBuildTime(); + +#endif http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5d9e13a6/be/src/exec/hdfs-lzo-text-scanner.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/hdfs-lzo-text-scanner.cc b/be/src/exec/hdfs-lzo-text-scanner.cc index 21e37c9..0ab775b 100644 --- a/be/src/exec/hdfs-lzo-text-scanner.cc +++ b/be/src/exec/hdfs-lzo-text-scanner.cc @@ -39,7 +39,7 @@ Status HdfsLzoTextScanner::library_load_status_; SpinLock HdfsLzoTextScanner::lzo_load_lock_; -const char* (*GetImpalaBuildVersion)(); +const char* (*GetImpalaLzoBuildVersion)(); HdfsScanner* (*HdfsLzoTextScanner::CreateLzoTextScanner)( HdfsScanNode* scan_node, RuntimeState* state); @@ -68,7 +68,7 @@ Status HdfsLzoTextScanner::IssueInitialRanges(HdfsScanNode* scan_node, if (!library_load_status_.ok()) { stringstream ss; ss << "Error loading impala-lzo library. Check that the impala-lzo library " - << "is at version " << IMPALA_BUILD_VERSION; + << "is at version " << GetDaemonBuildVersion(); library_load_status_.AddDetail(ss.str()); return library_load_status_; } @@ -85,13 +85,13 @@ Status HdfsLzoTextScanner::LoadLzoLibrary() { void* handle; RETURN_IF_ERROR(DynamicOpen(LIB_IMPALA_LZO.c_str(), &handle)); RETURN_IF_ERROR(DynamicLookup(handle, - "GetImpalaBuildVersion", reinterpret_cast<void**>(&GetImpalaBuildVersion))); + "GetImpalaBuildVersion", reinterpret_cast<void**>(&GetImpalaLzoBuildVersion))); - if (strcmp((*GetImpalaBuildVersion)(), IMPALA_BUILD_VERSION) != 0) { + if (strcmp((*GetImpalaLzoBuildVersion)(), GetDaemonBuildVersion()) != 0) { stringstream ss; ss << "Impala LZO library was built against Impala version " - << (*GetImpalaBuildVersion)() << ", but the running Impala version is " - << IMPALA_BUILD_VERSION; + << (*GetImpalaLzoBuildVersion)() << ", but the running Impala version is " + << GetDaemonBuildVersion(); if (FLAGS_skip_lzo_version_check) { LOG(ERROR) << ss.str(); } else { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5d9e13a6/be/src/exec/hdfs-parquet-table-writer.cc ---------------------------------------------------------------------- diff --git a/be/src/exec/hdfs-parquet-table-writer.cc b/be/src/exec/hdfs-parquet-table-writer.cc index 6193da7..87e42fb 100644 --- a/be/src/exec/hdfs-parquet-table-writer.cc +++ b/be/src/exec/hdfs-parquet-table-writer.cc @@ -633,8 +633,8 @@ Status HdfsParquetTableWriter::Init() { file_metadata_.version = PARQUET_CURRENT_VERSION; stringstream created_by; - created_by << "impala version " << IMPALA_BUILD_VERSION - << " (build " << IMPALA_BUILD_HASH << ")"; + created_by << "impala version " << GetDaemonBuildVersion() + << " (build " << GetDaemonBuildHash() << ")"; file_metadata_.__set_created_by(created_by.str()); // Default to snappy compressed http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5d9e13a6/be/src/service/impala-hs2-server.cc ---------------------------------------------------------------------- diff --git a/be/src/service/impala-hs2-server.cc b/be/src/service/impala-hs2-server.cc index 9bb9a56..803242e 100644 --- a/be/src/service/impala-hs2-server.cc +++ b/be/src/service/impala-hs2-server.cc @@ -705,7 +705,7 @@ void ImpalaServer::GetInfo(TGetInfoResp& return_val, return_val.infoValue.__set_stringValue("Impala"); break; case TGetInfoType::CLI_DBMS_VER: - return_val.infoValue.__set_stringValue(IMPALA_BUILD_VERSION); + return_val.infoValue.__set_stringValue(GetDaemonBuildVersion()); break; default: HS2_RETURN_ERROR(return_val, "Unsupported operation", http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5d9e13a6/be/src/util/debug-util.cc ---------------------------------------------------------------------- diff --git a/be/src/util/debug-util.cc b/be/src/util/debug-util.cc index 4cf606b..c342965 100644 --- a/be/src/util/debug-util.cc +++ b/be/src/util/debug-util.cc @@ -283,16 +283,16 @@ string PrintNumericPath(const SchemaPath& path) { string GetBuildVersion(bool compact) { stringstream ss; - ss << IMPALA_BUILD_VERSION + ss << GetDaemonBuildVersion() #ifdef NDEBUG << " RELEASE" #else << " DEBUG" #endif - << " (build " << IMPALA_BUILD_HASH + << " (build " << GetDaemonBuildHash() << ")"; if (!compact) { - ss << endl << "Built on " << IMPALA_BUILD_TIME; + ss << endl << "Built on " << GetDaemonBuildTime(); } return ss.str(); } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5d9e13a6/bin/gen_build_version.py ---------------------------------------------------------------------- diff --git a/bin/gen_build_version.py b/bin/gen_build_version.py index b2715bb..db8030e 100755 --- a/bin/gen_build_version.py +++ b/bin/gen_build_version.py @@ -21,31 +21,23 @@ import time; import filecmp from commands import getstatusoutput from time import localtime, strftime -from optparse import OptionParser - -parser = OptionParser() -parser.add_option("--noclean", action="store_true", default=False, - help="If specified, does not remove existing files and only replaces " - "them with freshly generated ones if they have changed.") -options, args = parser.parse_args() IMPALA_HOME = os.environ['IMPALA_HOME'] SAVE_VERSION_SCRIPT = os.path.join(IMPALA_HOME, 'bin/save-version.sh') VERSION_FILE_NAME = os.path.join(IMPALA_HOME, 'bin/version.info') -VERSION_HEADER_FILE_NAME = os.path.join(IMPALA_HOME, 'be/src/common/version.h') +VERSION_CC_FILE_NAME = os.path.join(IMPALA_HOME, 'be/src/common/version.cc') -# Remove existing version files only if --noclean was not specified. +# Remove existing version files only if they exist. # TODO: Might be useful to make a common utility function remove_if_clean. -if not options.noclean and os.path.isfile(VERSION_FILE_NAME): +if os.path.isfile(VERSION_FILE_NAME): print 'Removing existing file: %s' % (VERSION_FILE_NAME) os.remove(VERSION_FILE_NAME) -if not options.noclean and os.path.isfile(VERSION_HEADER_FILE_NAME): - print 'Removing existing file: %s' % (VERSION_HEADER_FILE_NAME) - os.remove(VERSION_HEADER_FILE_NAME) +if os.path.isfile(VERSION_CC_FILE_NAME): + print 'Removing existing file: %s' % (VERSION_CC_FILE_NAME) + os.remove(VERSION_CC_FILE_NAME) -# Generate a new version file only if there is no existing one. -if not os.path.isfile(VERSION_FILE_NAME): - os.system(SAVE_VERSION_SCRIPT) +# Generate a new version file. +os.system(SAVE_VERSION_SCRIPT) # version.info file has the format: # VERSION: <version> @@ -82,21 +74,29 @@ file_contents = """ // This is a generated file, DO NOT EDIT IT. // To change this file, see impala/bin/gen_build_version.py -#ifndef IMPALA_COMMON_VERSION_H -#define IMPALA_COMMON_VERSION_H +#include "common/version.h" #define IMPALA_BUILD_VERSION "%(build_version)s" #define IMPALA_BUILD_HASH "%(build_hash)s" #define IMPALA_BUILD_TIME "%(build_time)s" -#endif +const char* GetDaemonBuildVersion() { + return IMPALA_BUILD_VERSION; +} + +const char* GetDaemonBuildHash() { + return IMPALA_BUILD_HASH; +} + +const char* GetDaemonBuildTime() { + return IMPALA_BUILD_TIME; +} """ % {'build_version': version, 'build_hash': git_hash, 'build_time': build_time} file_contents = file_contents.strip() -# Generate a new version file only if there is no existing one. -if not os.path.isfile(VERSION_HEADER_FILE_NAME): - version_file = open(VERSION_HEADER_FILE_NAME, "w") - version_file.write(file_contents) - version_file.close() +# Generate a new version file. +version_file = open(VERSION_CC_FILE_NAME, "w") +version_file.write(file_contents) +version_file.close() http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5d9e13a6/bin/make_impala.sh ---------------------------------------------------------------------- diff --git a/bin/make_impala.sh b/bin/make_impala.sh index 70da52b..f243d89 100755 --- a/bin/make_impala.sh +++ b/bin/make_impala.sh @@ -126,7 +126,7 @@ then ${MAKE_CMD} clean fi -$IMPALA_HOME/bin/gen_build_version.py --noclean +$IMPALA_HOME/bin/gen_build_version.py if [ $CMAKE_ONLY -eq 1 ]; then exit 0
