This is an automated email from the ASF dual-hosted git repository. awong pushed a commit to branch branch-1.13.x in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 5a66fffcdcfbb97ea471d6db05d61dcf4b1b7c22 Author: Grant Henke <[email protected]> AuthorDate: Thu Nov 12 19:05:11 2020 -0600 [dist_test] Ship security libraries to dist_test This patch adjusts the lib whitelist to allow shipping the security libraries to dist_test and allow more flexibility when versions do not match the dist_test images versions. This was already happening for rhel6 installs due to the rhel6 workaround script linked below. With this change the libraries will be shipped even when not in thirdparty. https://github.com/apache/kudu/blob/master/thirdparty/install-openssl-el6-workaround.sh I also needed to adjust run_dist_test.py in order to set the SASL_PATH environment variable if SASL modules are present. Otherwise the system modules were still used. Change-Id: Id10afab6e9c48b9ffcf0da905993c7f2a1e606a6 Reviewed-on: http://gerrit.cloudera.org:8080/16716 Tested-by: Kudu Jenkins Reviewed-by: Andrew Wong <[email protected]> Reviewed-by: Alexey Serbin <[email protected]> (cherry picked from commit fab3a38d1da0d5d13d8e9c91fd306e03b65da4e6) Reviewed-on: http://gerrit.cloudera.org:8080/16802 Reviewed-by: Grant Henke <[email protected]> Tested-by: Andrew Wong <[email protected]> --- build-support/dist_test.py | 63 +++++++++++++++++++++++++++++++++++++-- build-support/run_dist_test.py | 6 ++++ cmake_modules/FindCyrusSASL.cmake | 2 +- 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/build-support/dist_test.py b/build-support/dist_test.py index 1696ea8..89e53bf 100755 --- a/build-support/dist_test.py +++ b/build-support/dist_test.py @@ -240,6 +240,11 @@ def get_test_executions(tests_regex, extra_args=None): def is_lib_whitelisted(lib): # No need to ship things like libc, libstdcxx, etc. if lib.startswith("/lib") or lib.startswith("/usr"): + # Ship the dynamically linked security libraries from + # OpenSSL and Cyrus SASL to better support submitting + # installed versions different from the dist_test image. + if "libcrypto" in lib or "libsasl2" in lib or "libssl" in lib: + return True return False return True @@ -262,8 +267,41 @@ def get_base_deps(dep_extractor): # of the test executable. We must include those dependencies in the archive # for the binaries to be usable. deps.extend(dep_extractor.extract_deps(d)) + + add_sasl_module_deps(deps) return deps +def add_sasl_module_deps(deps): + """ + The SASL module dependencies are used at runtime but are not discovered + via ldd in the dep_extractor. This method finds the sasl2 directory + relative to the libsasl2 library and adds all the libraries in that + directory. + """ + # Find the libsasl2 module in the dependencies. + sasl_lib = None + for dep in deps: + if "libsasl2" in dep: + sasl_lib = dep + break + + # Look for libplain in potential sasl2 module paths, which is required for + # Kudu's basic operation. + sasl_path = None + if sasl_lib: + path = os.path.join(os.path.dirname(sasl_lib), "sasl2") + if os.path.exists(path): + children = os.listdir(path) + for child in children: + if "libplain" in child: + sasl_path = path + break + + if sasl_path: + for dirpath, subdirs, files in os.walk(sasl_path): + for f in files: + dep = os.path.join(dirpath, f) + deps.append(dep) def is_outside_of_tree(path): repo_dir = rel_to_abs("./") @@ -282,7 +320,18 @@ def copy_system_library(lib): sys_lib_dir = rel_to_abs("build/dist-test-system-libs") if not os.path.exists(sys_lib_dir): os.makedirs(sys_lib_dir) - dst = os.path.join(sys_lib_dir, os.path.basename(lib)) + + sasl_dir = os.path.join(sys_lib_dir, "sasl2") + if not os.path.exists(sasl_dir): + os.makedirs(sasl_dir) + + # If the library is a SASL module keep it in its own directory so + # we can set the SASL_PATH environment variable in run_dist_test.py. + if "/sasl2/" in lib: + dst = os.path.join(sasl_dir, os.path.basename(lib)) + else: + dst = os.path.join(sys_lib_dir, os.path.basename(lib)) + # Copy if it doesn't exist, or the mtimes don't match. # Using shutil.copy2 preserves the mtime after the copy (like cp -p) if not os.path.exists(dst) or os.stat(dst).st_mtime != os.stat(lib).st_mtime: @@ -664,7 +713,17 @@ def add_java_subparser(subparsers): loop.set_defaults(func=loop_java_test) def dump_base_deps(parser, options): - print(json.dumps(get_base_deps(create_dependency_extractor()))) + deps = get_base_deps(create_dependency_extractor()) + relocated_deps = [] + # Deduplicate dependencies included via DEPS_FOR_ALL. + for d in set(deps): + # System libraries will end up being relative paths out + # of the build tree. We need to copy those into the build + # tree somewhere. + if is_outside_of_tree(d): + d = copy_system_library(d) + relocated_deps.append(d) + print(json.dumps(relocated_deps)) def add_internal_commands(subparsers): p = subparsers.add_parser('internal', help="[Internal commands not for users]") diff --git a/build-support/run_dist_test.py b/build-support/run_dist_test.py index cca701b..8d8a5a5 100755 --- a/build-support/run_dist_test.py +++ b/build-support/run_dist_test.py @@ -183,6 +183,12 @@ def main(): [os.path.join(ROOT, "build/dist-test-system-libs/")] + glob.glob(os.path.abspath(os.path.join(ROOT, "build/*/lib")))) + # If SASL modules are included in the dist-test-system-libs, set the + # SASL_PATH environment variable to use them instead of the system ones. + sasl_dir = os.path.join(ROOT, "build/dist-test-system-libs/sasl2") + if os.path.exists(sasl_dir): + env['SASL_PATH'] = sasl_dir + # Don't pollute /tmp in dist-test setting. If a test crashes, the dist-test slave # will clear up our working directory but won't be able to find and clean up things # left in /tmp. diff --git a/cmake_modules/FindCyrusSASL.cmake b/cmake_modules/FindCyrusSASL.cmake index 8f92cf5..24192b7 100644 --- a/cmake_modules/FindCyrusSASL.cmake +++ b/cmake_modules/FindCyrusSASL.cmake @@ -24,7 +24,7 @@ # # N.B: we do _not_ include sasl in thirdparty, for a fairly subtle reason. The # TLDR version is that newer versions of cyrus-sasl (>=2.1.26) have a bug fix -# for https://bugzilla.cyrusimap.org/show_bug.cgi?id=3590, but that bug fix +# for https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=728332, but that bug fix # relied on a change both on the plugin side and on the library side. If you # then try to run the new version of sasl (e.g from our thirdparty tree) with # an older version of a plugin (eg from RHEL6 install), you'll get a SASL_NOMECH
