Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package aws-efs-utils for openSUSE:Factory checked in at 2022-12-03 10:03:29 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/aws-efs-utils (Old) and /work/SRC/openSUSE:Factory/.aws-efs-utils.new.1835 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "aws-efs-utils" Sat Dec 3 10:03:29 2022 rev:12 rq:1039570 version:1.34.3 Changes: -------- --- /work/SRC/openSUSE:Factory/aws-efs-utils/aws-efs-utils.changes 2022-10-20 11:12:00.596055941 +0200 +++ /work/SRC/openSUSE:Factory/.aws-efs-utils.new.1835/aws-efs-utils.changes 2022-12-03 10:03:37.547236717 +0100 @@ -1,0 +2,14 @@ +Fri Dec 2 11:38:36 UTC 2022 - John Paul Adrian Glaubitz <[email protected]> + +- Update to version 1.34.3 + * Fix stunnel constantly restart issue when upgrading + from 1.32.1 and before version to latest version + * Fix race in stunnel port selection + * Disable journal entry fetch from systemctl call +- from version 1.34.2 + * Fix potential issue on AL2 when watchdog trying to restart + stunnel for the TLS mounts that existing before upgrade +- from version 1.34.1 + * Update Amazon Linux 2 platform to use namespaced stunnel5 + +------------------------------------------------------------------- Old: ---- efs-utils-1.33.4.tar.gz New: ---- efs-utils-1.34.3.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ aws-efs-utils.spec ++++++ --- /var/tmp/diff_new_pack.uspgF3/_old 2022-12-03 10:03:38.111239851 +0100 +++ /var/tmp/diff_new_pack.uspgF3/_new 2022-12-03 10:03:38.119239896 +0100 @@ -17,7 +17,7 @@ Name: aws-efs-utils -Version: 1.33.4 +Version: 1.34.3 Release: 0 Summary: Utilities for using the EFS file systems License: MIT ++++++ efs-utils-1.33.4.tar.gz -> efs-utils-1.34.3.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/efs-utils-1.33.4/amazon-efs-utils.spec new/efs-utils-1.34.3/amazon-efs-utils.spec --- old/efs-utils-1.33.4/amazon-efs-utils.spec 2022-09-22 07:36:56.000000000 +0200 +++ new/efs-utils-1.34.3/amazon-efs-utils.spec 2022-12-02 12:17:58.000000000 +0100 @@ -35,7 +35,7 @@ %endif Name : amazon-efs-utils -Version : 1.33.4 +Version : 1.34.3 Release : 1%{platform} Summary : This package provides utilities for simplifying the use of EFS file systems @@ -43,13 +43,15 @@ License : MIT URL : https://aws.amazon.com/efs -Packager : Amazon.com, Inc. <http://aws.amazon.com> -Vendor : Amazon.com BuildArch : noarch Requires : nfs-utils +%if 0%{?amzn2} +Requires : stunnel5 +%else Requires : stunnel >= 4.56 +%endif Requires : %{python_requires} Requires : openssl >= 1.0.2 Requires : util-linux @@ -135,6 +137,17 @@ %clean %changelog +* Thu Dec 1 2022 Preetham Puneeth Munipalli <[email protected]> - 1.34.3 +- Fix potential tlsport selection race condition by closing socket right before establishing stunnel +- Fix stunnel constantly restart issue when upgrading from 1.32.1 and before version to latest version +- Speed up the way to check network availability by using systemctl is-active + +* Tue Nov 22 2022 Preetham Puneeth Munipalli <[email protected]> - 1.34.2 +- Fix potential issue on AL2 when watchdog trying to restart stunnel for the TLS mounts that existing before upgrade + +* Thu Sep 29 2022 Preetham Puneeth Munipalli <[email protected]> - 1.34.1 +- Update Amazon Linux 2 platform to use namespaced stunnel5 + * Thu Sep 1 2022 Yuan Gao <[email protected]> - 1.33.4 - Fix potential issue where watchdog sending signal to incorrect processes. - Add support for enabling FIPS mode for both stunnel and AWS API calls. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/efs-utils-1.33.4/build-deb.sh new/efs-utils-1.34.3/build-deb.sh --- old/efs-utils-1.33.4/build-deb.sh 2022-09-22 07:36:56.000000000 +0200 +++ new/efs-utils-1.34.3/build-deb.sh 2022-12-02 12:17:58.000000000 +0100 @@ -11,7 +11,7 @@ BASE_DIR=$(pwd) BUILD_ROOT=${BASE_DIR}/build/debbuild -VERSION=1.33.4 +VERSION=1.34.3 RELEASE=1 DEB_SYSTEM_RELEASE_PATH=/etc/os-release diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/efs-utils-1.33.4/config.ini new/efs-utils-1.34.3/config.ini --- old/efs-utils-1.33.4/config.ini 2022-09-22 07:36:56.000000000 +0200 +++ new/efs-utils-1.34.3/config.ini 2022-12-02 12:17:58.000000000 +0100 @@ -7,5 +7,5 @@ # [global] -version=1.33.4 +version=1.34.3 release=1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/efs-utils-1.33.4/dist/amazon-efs-utils.control new/efs-utils-1.34.3/dist/amazon-efs-utils.control --- old/efs-utils-1.33.4/dist/amazon-efs-utils.control 2022-09-22 07:36:56.000000000 +0200 +++ new/efs-utils-1.34.3/dist/amazon-efs-utils.control 2022-12-02 12:17:58.000000000 +0100 @@ -1,6 +1,6 @@ Package: amazon-efs-utils Architecture: all -Version: 1.33.4 +Version: 1.34.3 Section: utils Depends: python3, nfs-common, stunnel4 (>= 4.56), openssl (>= 1.0.2), util-linux Priority: optional diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/efs-utils-1.33.4/src/mount_efs/__init__.py new/efs-utils-1.34.3/src/mount_efs/__init__.py --- old/efs-utils-1.33.4/src/mount_efs/__init__.py 2022-09-22 07:36:56.000000000 +0200 +++ new/efs-utils-1.34.3/src/mount_efs/__init__.py 2022-12-02 12:17:58.000000000 +0100 @@ -85,9 +85,16 @@ BOTOCORE_PRESENT = False -VERSION = "1.33.4" +VERSION = "1.34.3" SERVICE = "elasticfilesystem" +AMAZON_LINUX_2_RELEASE_ID = "Amazon Linux release 2 (Karoo)" +AMAZON_LINUX_2_PRETTY_NAME = "Amazon Linux 2" +AMAZON_LINUX_2_RELEASE_VERSIONS = [ + AMAZON_LINUX_2_RELEASE_ID, + AMAZON_LINUX_2_PRETTY_NAME, +] + CLONE_NEWNET = 0x40000000 CONFIG_FILE = "/etc/amazon/efs/efs-utils.conf" CONFIG_SECTION = "mount" @@ -932,28 +939,25 @@ return lower_bound, upper_bound -def choose_tls_port(config, options): +def choose_tls_port_and_get_bind_sock(config, options): if "tlsport" in options: ports_to_try = [int(options["tlsport"])] else: lower_bound, upper_bound = get_tls_port_range(config) - tls_ports = list(range(lower_bound, upper_bound)) - - # Choose a random midpoint, and then try ports in-order from there - mid = random.randrange(len(tls_ports)) + ports_to_try = list(range(lower_bound, upper_bound)) - ports_to_try = tls_ports[mid:] + tls_ports[:mid] - assert len(tls_ports) == len(ports_to_try) + # shuffle the ports_to_try to reduce possibility of multiple mounts starting from same port range + random.shuffle(ports_to_try) if "netns" not in options: - tls_port = find_tls_port_in_range(ports_to_try) + tls_port_sock = find_tls_port_in_range_and_get_bind_sock(ports_to_try) else: with NetNS(nspath=options["netns"]): - tls_port = find_tls_port_in_range(ports_to_try) + tls_port_sock = find_tls_port_in_range_and_get_bind_sock(ports_to_try) - if tls_port: - return tls_port + if tls_port_sock: + return tls_port_sock if "tlsport" in options: fatal_error( @@ -967,14 +971,13 @@ ) -def find_tls_port_in_range(ports_to_try): +def find_tls_port_in_range_and_get_bind_sock(ports_to_try): sock = socket.socket() for tls_port in ports_to_try: try: logging.info("binding %s", tls_port) sock.bind(("localhost", tls_port)) - sock.close() - return tls_port + return sock except socket.error as e: logging.info(e) continue @@ -1095,31 +1098,32 @@ def _stunnel_bin(): - return find_command_path( - "stunnel", - "Please install it following the instructions at " - "https://docs.aws.amazon.com/efs/latest/ug/using-amazon-efs-utils.html#upgrading-stunnel";, - ) + installation_message = "Please install it following the instructions at: https://docs.aws.amazon.com/efs/latest/ug/using-amazon-efs-utils.html#upgrading-stunnel"; + if get_system_release_version() in AMAZON_LINUX_2_RELEASE_VERSIONS: + return find_command_path("stunnel5", installation_message) + else: + return find_command_path("stunnel", installation_message) def find_command_path(command, install_method): + # If not running on macOS, use linux paths + if not check_if_platform_is_mac(): + env_path = ( + "/sbin:/usr/sbin:/usr/local/sbin:/root/bin:/usr/local/bin:/usr/bin:/bin" + ) + # Homebrew on x86 macOS uses /usr/local/bin; Homebrew on Apple Silicon macOS uses /opt/homebrew/bin since v3.0.0 + # For more information, see https://brew.sh/2021/02/05/homebrew-3.0.0/ + else: + env_path = "/opt/homebrew/bin:/usr/local/bin" + os.putenv("PATH", env_path) + try: - # If not running on macOS, use linux paths - if not check_if_platform_is_mac(): - env_path = ( - "/sbin:/usr/sbin:/usr/local/sbin:/root/bin:/usr/local/bin:/usr/bin:/bin" - ) - # Homebrew on x86 macOS uses /usr/local/bin; Homebrew on Apple Silicon macOS uses /opt/homebrew/bin since v3.0.0 - # For more information, see https://brew.sh/2021/02/05/homebrew-3.0.0/ - else: - env_path = "/opt/homebrew/bin:/usr/local/bin" - os.putenv("PATH", env_path) path = subprocess.check_output(["which", command]) + return path.strip().decode() except subprocess.CalledProcessError as e: fatal_error( "Failed to locate %s in %s - %s" % (command, env_path, install_method), e ) - return path.strip().decode() def get_system_release_version(): @@ -1342,7 +1346,7 @@ with open(os.devnull, "w") as devnull: if not check_if_platform_is_mac(): rc = subprocess.call( - ["systemctl", "status", "network.target"], + ["systemctl", "is-active", "network.target"], stdout=devnull, stderr=devnull, close_fds=True, @@ -1456,6 +1460,13 @@ raise +# Example of a localhost bind sock: sock.bind(('localhost',12345)) +# sock.getsockname() -> ('127.0.0.1', 12345) +# This function returns the port of the bind socket, in the example is 12345 +def get_tls_port_from_sock(tls_port_sock): + return tls_port_sock.getsockname()[1] + + @contextmanager def bootstrap_tls( config, @@ -1467,85 +1478,93 @@ state_file_dir=STATE_FILE_DIR, fallback_ip_address=None, ): - tls_port = choose_tls_port(config, options) - # override the tlsport option so that we can later override the port the NFS client uses to connect to stunnel. - # if the user has specified tlsport=X at the command line this will just re-set tlsport to X. - options["tlsport"] = tls_port - - use_iam = "iam" in options - ap_id = options.get("accesspoint") - cert_details = {} - security_credentials = None - client_info = get_client_info(config) - region = get_target_region(config) - - if use_iam: - aws_creds_uri = options.get("awscredsuri") - if aws_creds_uri: - kwargs = {"aws_creds_uri": aws_creds_uri} - else: - kwargs = {"awsprofile": get_aws_profile(options, use_iam)} - - security_credentials, credentials_source = get_aws_security_credentials( - config, use_iam, region, **kwargs - ) + tls_port_sock = choose_tls_port_and_get_bind_sock(config, options) + tls_port = get_tls_port_from_sock(tls_port_sock) - if credentials_source: - cert_details["awsCredentialsMethod"] = credentials_source - - if ap_id: - cert_details["accessPoint"] = ap_id - - # additional symbol appended to avoid naming collisions - cert_details["mountStateDir"] = ( - get_mount_specific_filename(fs_id, mountpoint, tls_port) + "+" - ) - # common name for certificate signing request is max 64 characters - cert_details["commonName"] = socket.gethostname()[0:64] - region = get_target_region(config) - cert_details["region"] = region - cert_details["certificateCreationTime"] = create_certificate( - config, - cert_details["mountStateDir"], - cert_details["commonName"], - cert_details["region"], - fs_id, - security_credentials, - ap_id, - client_info, - base_path=state_file_dir, - ) - cert_details["certificate"] = os.path.join( - state_file_dir, cert_details["mountStateDir"], "certificate.pem" - ) - cert_details["privateKey"] = get_private_key_path() - cert_details["fsId"] = fs_id - - start_watchdog(init_system) + try: + # override the tlsport option so that we can later override the port the NFS client uses to connect to stunnel. + # if the user has specified tlsport=X at the command line this will just re-set tlsport to X. + options["tlsport"] = tls_port + + use_iam = "iam" in options + ap_id = options.get("accesspoint") + cert_details = {} + security_credentials = None + client_info = get_client_info(config) + region = get_target_region(config) + + if use_iam: + aws_creds_uri = options.get("awscredsuri") + if aws_creds_uri: + kwargs = {"aws_creds_uri": aws_creds_uri} + else: + kwargs = {"awsprofile": get_aws_profile(options, use_iam)} + + security_credentials, credentials_source = get_aws_security_credentials( + config, use_iam, region, **kwargs + ) + + if credentials_source: + cert_details["awsCredentialsMethod"] = credentials_source + + if ap_id: + cert_details["accessPoint"] = ap_id + + # additional symbol appended to avoid naming collisions + cert_details["mountStateDir"] = ( + get_mount_specific_filename(fs_id, mountpoint, tls_port) + "+" + ) + # common name for certificate signing request is max 64 characters + cert_details["commonName"] = socket.gethostname()[0:64] + region = get_target_region(config) + cert_details["region"] = region + cert_details["certificateCreationTime"] = create_certificate( + config, + cert_details["mountStateDir"], + cert_details["commonName"], + cert_details["region"], + fs_id, + security_credentials, + ap_id, + client_info, + base_path=state_file_dir, + ) + cert_details["certificate"] = os.path.join( + state_file_dir, cert_details["mountStateDir"], "certificate.pem" + ) + cert_details["privateKey"] = get_private_key_path() + cert_details["fsId"] = fs_id + + start_watchdog(init_system) - if not os.path.exists(state_file_dir): - create_required_directory(config, state_file_dir) + if not os.path.exists(state_file_dir): + create_required_directory(config, state_file_dir) - verify_level = int(options.get("verify", DEFAULT_STUNNEL_VERIFY_LEVEL)) - ocsp_enabled = is_ocsp_enabled(config, options) + verify_level = int(options.get("verify", DEFAULT_STUNNEL_VERIFY_LEVEL)) + ocsp_enabled = is_ocsp_enabled(config, options) - stunnel_config_file = write_stunnel_config_file( - config, - state_file_dir, - fs_id, - mountpoint, - tls_port, - dns_name, - verify_level, - ocsp_enabled, - options, - region, - cert_details=cert_details, - fallback_ip_address=fallback_ip_address, - ) - tunnel_args = [_stunnel_bin(), stunnel_config_file] - if "netns" in options: - tunnel_args = ["nsenter", "--net=" + options["netns"]] + tunnel_args + stunnel_config_file = write_stunnel_config_file( + config, + state_file_dir, + fs_id, + mountpoint, + tls_port, + dns_name, + verify_level, + ocsp_enabled, + options, + region, + cert_details=cert_details, + fallback_ip_address=fallback_ip_address, + ) + tunnel_args = [_stunnel_bin(), stunnel_config_file] + if "netns" in options: + tunnel_args = ["nsenter", "--net=" + options["netns"]] + tunnel_args + finally: + # Always close the socket we created when choosing TLS port only until now to + # 1. avoid concurrent TLS mount port collision 2. enable stunnel process to bind the port + logging.debug("Closing socket used to choose TLS port %s.", tls_port) + tls_port_sock.close() # launch the tunnel in a process group so if it has any child processes, they can be killed easily by the mount watchdog logging.info('Starting TLS tunnel: "%s"', " ".join(tunnel_args)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/efs-utils-1.33.4/src/watchdog/__init__.py new/efs-utils-1.34.3/src/watchdog/__init__.py --- old/efs-utils-1.33.4/src/watchdog/__init__.py 2022-09-22 07:36:56.000000000 +0200 +++ new/efs-utils-1.34.3/src/watchdog/__init__.py 2022-12-02 12:17:58.000000000 +0100 @@ -15,6 +15,7 @@ import logging import logging.handlers import os +import platform import pwd import re import shutil @@ -49,7 +50,13 @@ from urllib2 import HTTPError, HTTPHandler, Request, URLError, build_opener, urlopen -VERSION = "1.33.4" +AMAZON_LINUX_2_RELEASE_ID = "Amazon Linux release 2 (Karoo)" +AMAZON_LINUX_2_PRETTY_NAME = "Amazon Linux 2" +AMAZON_LINUX_2_RELEASE_VERSIONS = [ + AMAZON_LINUX_2_RELEASE_ID, + AMAZON_LINUX_2_PRETTY_NAME, +] +VERSION = "1.34.3" SERVICE = "elasticfilesystem" CONFIG_FILE = "/etc/amazon/efs/efs-utils.conf" @@ -169,6 +176,11 @@ # Default unmount count for consistency DEFAULT_UNMOUNT_COUNT_FOR_CONSISTENCY = 5 +MAC_OS_PLATFORM_LIST = ["darwin"] +SYSTEM_RELEASE_PATH = "/etc/system-release" +OS_RELEASE_PATH = "/etc/os-release" +STUNNEL_INSTALLATION_MESSAGE = "Please install it following the instructions at: https://docs.aws.amazon.com/efs/latest/ug/using-amazon-efs-utils.html#upgrading-stunnel"; + def fatal_error(user_message, log_message=None): if log_message is None: @@ -556,11 +568,7 @@ ) return resp_dict - except ValueError as e: - logging.info( - 'ValueError parsing "%s" into json: %s. Returning response body.' - % (str(resp_body), e) - ) + except ValueError: return resp_body if resp_body_type is str else resp_body.decode("utf-8") @@ -817,13 +825,16 @@ return False pid_in_stunnel_pid_file = get_pid_in_state_dir(state_file, state_file_dir) + # efs-utils versions older than 1.32.2 does not create a pid file in state dir + # To avoid the healthy stunnel established by those version to be treated as not running due to the missing pid file, which can result in stunnel being constantly restarted, + # assuming the stunnel is still running even if the stunnel pid file does not exist. if not pid_in_stunnel_pid_file: logging.debug( - "Pid file of stunnel is already removed for %s, assuming the stunnel with pid %s is no longer running.", + "Pid file of stunnel does not exist for %s. It is possible that the stunnel is no longer running or the mount was mounted using an older version efs-utils (<1.32.2). Assuming the stunnel with pid %s is still running.", state_file, state_pid, ) - return False + elif int(state_pid) != int(pid_in_stunnel_pid_file): logging.warning( "Stunnel pid mismatch in state file (pid = %s) and stunnel pid file (pid = %s). Assuming the " @@ -847,18 +858,130 @@ return False -def start_tls_tunnel(child_procs, state_file, command): +def check_if_platform_is_mac(): + return sys.platform in MAC_OS_PLATFORM_LIST + + +def get_system_release_version(): + # MacOS does not maintain paths /etc/os-release and /etc/sys-release + if check_if_platform_is_mac(): + return platform.platform() + + try: + with open(SYSTEM_RELEASE_PATH) as f: + return f.read().strip() + except IOError: + logging.debug("Unable to read %s", SYSTEM_RELEASE_PATH) + + try: + with open(OS_RELEASE_PATH) as f: + for line in f: + if "PRETTY_NAME" in line: + return line.split("=")[1].strip() + except IOError: + logging.debug("Unable to read %s", OS_RELEASE_PATH) + + return DEFAULT_UNKNOWN_VALUE + + +def find_command_path(command, install_method): + # If not running on macOS, use linux paths + if not check_if_platform_is_mac(): + env_path = ( + "/sbin:/usr/sbin:/usr/local/sbin:/root/bin:/usr/local/bin:/usr/bin:/bin" + ) + # Homebrew on x86 macOS uses /usr/local/bin; Homebrew on Apple Silicon macOS uses /opt/homebrew/bin since v3.0.0 + # For more information, see https://brew.sh/2021/02/05/homebrew-3.0.0/ + else: + env_path = "/opt/homebrew/bin:/usr/local/bin" + os.putenv("PATH", env_path) + + try: + path = subprocess.check_output(["which", command]) + return path.strip().decode() + except subprocess.CalledProcessError as e: + fatal_error( + "Failed to locate %s in %s - %s" % (command, env_path, install_method), e + ) + + +# In ECS amazon linux 2, we start stunnel using `nsenter` which will run as a subprocess of bash, utilizes the `setns` +# system call to join an existing namespace and then executes the specified program using `exec`. Any exception won't +# be caught properly by subprocess. +# As a precaution on ECS AL2 that stunnel bin is removed after installing new efs-utils, and watchdog cannot launch +# stunnel for previous old mount, we do a replacement of stunnel path in the command to the stunnel5 path. +# +def update_stunnel_command_for_ecs_amazon_linux_2( + command, state, state_file_dir, state_file +): + if ( + "nsenter" in command + and "stunnel5" not in " ".join(command) + and get_system_release_version() in AMAZON_LINUX_2_RELEASE_VERSIONS + ): + for i in range(len(command)): + if "stunnel" in command[i] and "stunnel-config" not in command[i]: + command[i] = find_command_path("stunnel5", STUNNEL_INSTALLATION_MESSAGE) + break + logging.info( + "Rewriting %s with new stunnel cmd: %s for ECS Amazon Linux 2 platform.", + state_file, + " ".join(state["cmd"]), + ) + rewrite_state_file(state, state_file_dir, state_file) + return command + + +def start_tls_tunnel(child_procs, state, state_file_dir, state_file): # launch the tunnel in a process group so if it has any child processes, they can be killed easily + command = state["cmd"] logging.info('Starting TLS tunnel: "%s"', " ".join(command)) - tunnel = subprocess.Popen( - command, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - preexec_fn=os.setsid, - close_fds=True, + + command = update_stunnel_command_for_ecs_amazon_linux_2( + command, state, state_file_dir, state_file ) + tunnel = None + try: + tunnel = subprocess.Popen( + command, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + preexec_fn=os.setsid, + close_fds=True, + ) + except FileNotFoundError as e: + logging.warning("Watchdog failed to start stunnel due to %s", e) + + # https://github.com/kubernetes-sigs/aws-efs-csi-driver/issues/812 It is possible that the stunnel is not + # present anymore and replaced by stunnel5 on AL2, meanwhile watchdog is attempting to restart stunnel for + # mount using old efs-utils based on old state file generated during previous mount, which has stale command + # using stunnel bin. Update the state file if the stunnel does not exist anymore, and use stunnel5 on Al2. + # + if get_system_release_version() in AMAZON_LINUX_2_RELEASE_VERSIONS: + for i in range(len(command)): + if "stunnel" in command[i] and "stunnel-config" not in command[i]: + command[i] = find_command_path( + "stunnel5", STUNNEL_INSTALLATION_MESSAGE + ) + break + + tunnel = subprocess.Popen( + command, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + preexec_fn=os.setsid, + close_fds=True, + ) + + state["cmd"] = command + logging.info( + "Rewriting %s with new stunnel cmd: %s for Amazon Linux 2 platform.", + state_file, + " ".join(state["cmd"]), + ) + rewrite_state_file(state, state_file_dir, state_file) - if not is_pid_running(tunnel.pid): + if tunnel is None or not is_pid_running(tunnel.pid): fatal_error( "Failed to initialize TLS tunnel for %s" % state_file, "Failed to start TLS tunnel.", @@ -940,7 +1063,7 @@ ) return - new_tunnel_pid = start_tls_tunnel(child_procs, state_file, state["cmd"]) + new_tunnel_pid = start_tls_tunnel(child_procs, state, state_file_dir, state_file) state["pid"] = new_tunnel_pid logging.debug("Rewriting %s with new pid: %d", state_file, new_tunnel_pid) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/efs-utils-1.33.4/test/mount_efs_test/test_bootstrap_tls.py new/efs-utils-1.34.3/test/mount_efs_test/test_bootstrap_tls.py --- old/efs-utils-1.33.4/test/mount_efs_test/test_bootstrap_tls.py 2022-09-22 07:36:56.000000000 +0200 +++ new/efs-utils-1.34.3/test/mount_efs_test/test_bootstrap_tls.py 2022-12-02 12:17:58.000000000 +0100 @@ -17,6 +17,7 @@ MOUNT_POINT = "/mnt" REGION = "us-east-1" + DEFAULT_TLS_PORT = 20049 EXPECTED_STUNNEL_CONFIG_FILE_BASE = "stunnel-config.fs-deadbeef.mnt." @@ -171,6 +172,11 @@ state_file_dir = str(tmpdir) tls_port = 1000 + tls_port_sock_mock = MagicMock() + tls_port_sock_mock.getsockname.return_value = ("local_host", tls_port) + tls_port_sock_mock.close.side_effect = None + mocker.patch("socket.socket", return_value=tls_port_sock_mock) + mocker.patch("mount_efs._stunnel_bin", return_value="/usr/bin/stunnel") with mount_efs.bootstrap_tls( MOCK_CONFIG, @@ -189,7 +195,11 @@ assert "/usr/bin/stunnel" in popen_args assert EXPECTED_STUNNEL_CONFIG_FILE in popen_args - assert 1000 == write_config_args[4] # positional argument for tls_port + assert tls_port == write_config_args[4] # positional argument for tls_port + # Ensure tls port socket is closed in bootstrap_tls + # The number is two here, the first one is the actual socket when choosing tls port, the second one is a socket to + # verify tls port can be connected after establishing TLS stunnel. They share the same mock. + assert 2 == tls_port_sock_mock.close.call_count def test_bootstrap_tls_non_default_verify_level(mocker, tmpdir): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/efs-utils-1.33.4/test/mount_efs_test/test_choose_tls_port.py new/efs-utils-1.34.3/test/mount_efs_test/test_choose_tls_port.py --- old/efs-utils-1.33.4/test/mount_efs_test/test_choose_tls_port.py 2022-09-22 07:36:56.000000000 +0200 +++ new/efs-utils-1.34.3/test/mount_efs_test/test_choose_tls_port.py 2022-12-02 12:17:58.000000000 +0100 @@ -4,6 +4,7 @@ # for the specific language governing permissions and limitations under # the License. +import random import socket from unittest.mock import MagicMock @@ -19,7 +20,10 @@ from configparser import ConfigParser DEFAULT_TLS_PORT_RANGE_LOW = 20049 -DEFAULT_TLS_PORT_RANGE_HIGH = 20449 +DEFAULT_TLS_PORT_RANGE_HIGH = 21049 +DEFAULT_TLS_PORT = random.randrange( + DEFAULT_TLS_PORT_RANGE_LOW, DEFAULT_TLS_PORT_RANGE_HIGH +) def _get_config(): @@ -42,25 +46,30 @@ def test_choose_tls_port_first_try(mocker): - mocker.patch("socket.socket", return_value=MagicMock()) + sock_mock = MagicMock() + sock_mock.getsockname.return_value = ("local_host", DEFAULT_TLS_PORT) + mocker.patch("socket.socket", return_value=sock_mock) options = {} - tls_port = mount_efs.choose_tls_port(_get_config(), options) - + tls_port_sock = mount_efs.choose_tls_port_and_get_bind_sock(_get_config(), options) + tls_port = mount_efs.get_tls_port_from_sock(tls_port_sock) assert DEFAULT_TLS_PORT_RANGE_LOW <= tls_port <= DEFAULT_TLS_PORT_RANGE_HIGH def test_choose_tls_port_second_try(mocker): bad_sock = MagicMock() bad_sock.bind.side_effect = [socket.error, None] + bad_sock.getsockname.return_value = ("local_host", DEFAULT_TLS_PORT) options = {} mocker.patch("socket.socket", return_value=bad_sock) - tls_port = mount_efs.choose_tls_port(_get_config(), options) + tls_port_sock = mount_efs.choose_tls_port_and_get_bind_sock(_get_config(), options) + tls_port = mount_efs.get_tls_port_from_sock(tls_port_sock) assert DEFAULT_TLS_PORT_RANGE_LOW <= tls_port <= DEFAULT_TLS_PORT_RANGE_HIGH assert 2 == bad_sock.bind.call_count + assert 1 == bad_sock.getsockname.call_count def test_choose_tls_port_never_succeeds(mocker, capsys): @@ -71,7 +80,7 @@ mocker.patch("socket.socket", return_value=bad_sock) with pytest.raises(SystemExit) as ex: - mount_efs.choose_tls_port(_get_config(), options) + mount_efs.choose_tls_port_and_get_bind_sock(_get_config(), options) assert 0 != ex.value.code @@ -85,12 +94,15 @@ def test_choose_tls_port_option_specified(mocker): - mocker.patch("socket.socket", return_value=MagicMock()) - options = {"tlsport": 1000} + sock_mock = MagicMock() + sock_mock.getsockname.return_value = ("local_host", DEFAULT_TLS_PORT) + mocker.patch("socket.socket", return_value=sock_mock) + options = {"tlsport": DEFAULT_TLS_PORT} - tls_port = mount_efs.choose_tls_port(_get_config(), options) + tls_port_sock = mount_efs.choose_tls_port_and_get_bind_sock(_get_config(), options) + tls_port = mount_efs.get_tls_port_from_sock(tls_port_sock) - assert 1000 == tls_port + assert DEFAULT_TLS_PORT == tls_port def test_choose_tls_port_option_specified_unavailable(mocker, capsys): @@ -101,7 +113,7 @@ mocker.patch("socket.socket", return_value=bad_sock) with pytest.raises(SystemExit) as ex: - mount_efs.choose_tls_port(_get_config(), options) + mount_efs.choose_tls_port_and_get_bind_sock(_get_config(), options) assert 0 != ex.value.code @@ -117,7 +129,7 @@ mocker.patch("socket.socket", return_value=MagicMock()) options = {"netns": "/proc/1000/ns/net"} - mount_efs.choose_tls_port(_get_config(), options) + mount_efs.choose_tls_port_and_get_bind_sock(_get_config(), options) utils.assert_called(setns_mock) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/efs-utils-1.33.4/test/mount_efs_test/test_helper_function.py new/efs-utils-1.34.3/test/mount_efs_test/test_helper_function.py --- old/efs-utils-1.33.4/test/mount_efs_test/test_helper_function.py 2022-09-22 07:36:56.000000000 +0200 +++ new/efs-utils-1.34.3/test/mount_efs_test/test_helper_function.py 2022-12-02 12:17:58.000000000 +0100 @@ -5,8 +5,10 @@ # the License. import logging +import sys +import unittest from collections import namedtuple -from unittest.mock import MagicMock +from unittest.mock import MagicMock, mock_open import pytest from botocore.exceptions import ProfileNotFound @@ -25,11 +27,13 @@ except ImportError: from urllib.error import HTTPError, URLError - DEFAULT_REGION = "us-east-1" ACCESS_KEY_ID_VAL = "FAKE_AWS_ACCESS_KEY_ID" SECRET_ACCESS_KEY_VAL = "FAKE_AWS_SECRET_ACCESS_KEY" SESSION_TOKEN_VAL = "FAKE_SESSION_TOKEN" +MACOS = "macOS" +AL2 = "Amazon Linux release 2" +NON_AL2_RELEASE_ID_VAL = "FAKE_NON_AL2_RELEASE_ID_VAL" def get_config( @@ -459,3 +463,78 @@ "profile", "test_profile" ) boto_session_mock.get_credentials.assert_called_once_with() + + +def test_get_system_release_version_macos(mocker): + mocker.patch("mount_efs.check_if_platform_is_mac", return_value=True) + platform_mock = mocker.patch("platform.platform", return_value=MACOS) + assert MACOS == mount_efs.get_system_release_version() + utils.assert_called_once(platform_mock) + + +def test_get_system_release_version_linux_read_from_sys_release_path(mocker): + mocker.patch("mount_efs.check_if_platform_is_mac", return_value=False) + open_mock = mocker.patch("builtins.open", mock_open(read_data=AL2)) + platform_mock = mocker.patch("platform.platform") + assert AL2 == mount_efs.get_system_release_version() + utils.assert_not_called(platform_mock) + utils.assert_called_once(open_mock) + + [email protected](sys.version_info[1] < 7, "Not supported in python3.6 and below.") +def test_get_system_release_version_linux_read_from_os_release_path(mocker): + mocker.patch("mount_efs.check_if_platform_is_mac", return_value=False) + mock = mock_open() + mock.side_effect = [ + FileNotFoundError, + mock_open(read_data="PRETTY_NAME=Amazon Linux release 2").return_value, + ] + open_mock = mocker.patch("builtins.open", mock) + platform_mock = mocker.patch("platform.platform") + assert AL2 == mount_efs.get_system_release_version() + utils.assert_not_called(platform_mock) + utils.assert_called_n_times(open_mock, 2) + + +def test_get_system_release_version_linux_unknown(mocker): + mocker.patch("mount_efs.check_if_platform_is_mac", return_value=False) + open_mock = mocker.patch("builtins.open", side_effect=FileNotFoundError) + platform_mock = mocker.patch("platform.platform") + assert mount_efs.DEFAULT_UNKNOWN_VALUE == mount_efs.get_system_release_version() + utils.assert_not_called(platform_mock) + utils.assert_called_n_times(open_mock, 2) + + +def test_stunnel5_al2_with_pretty_name(mocker): + check_output_mock = mocker.patch("subprocess.check_output") + mocker.patch( + "mount_efs.get_system_release_version", + return_value=mount_efs.AMAZON_LINUX_2_PRETTY_NAME, + ) + mount_efs._stunnel_bin() + args, _ = check_output_mock.call_args + args = args[0] + assert "stunnel5" == args[1] + + +def test_stunnel5_al2_with_release_id(mocker): + check_output_mock = mocker.patch("subprocess.check_output") + mocker.patch( + "mount_efs.get_system_release_version", + return_value=mount_efs.AMAZON_LINUX_2_RELEASE_ID, + ) + mount_efs._stunnel_bin() + args, _ = check_output_mock.call_args + args = args[0] + assert "stunnel5" == args[1] + + +def test_stunnel5_non_al2(mocker): + check_output_mock = mocker.patch("subprocess.check_output") + mocker.patch( + "mount_efs.get_system_release_version", return_value=NON_AL2_RELEASE_ID_VAL + ) + mount_efs._stunnel_bin() + args, _ = check_output_mock.call_args + args = args[0] + assert "stunnel" == args[1] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/efs-utils-1.33.4/test/watchdog_test/test_helper_function.py new/efs-utils-1.34.3/test/watchdog_test/test_helper_function.py --- old/efs-utils-1.33.4/test/watchdog_test/test_helper_function.py 2022-09-22 07:36:56.000000000 +0200 +++ new/efs-utils-1.34.3/test/watchdog_test/test_helper_function.py 2022-12-02 12:17:58.000000000 +0100 @@ -5,8 +5,10 @@ # the License. import logging +import sys +import unittest from collections import namedtuple -from unittest.mock import MagicMock +from unittest.mock import MagicMock, mock_open from botocore.exceptions import ProfileNotFound @@ -30,6 +32,8 @@ ACCESS_KEY_ID_VAL = "FAKE_AWS_ACCESS_KEY_ID" SECRET_ACCESS_KEY_VAL = "FAKE_AWS_SECRET_ACCESS_KEY" SESSION_TOKEN_VAL = "FAKE_SESSION_TOKEN" +MACOS = "macOS" +AL2 = "Amazon Linux release 2" def get_config( @@ -397,3 +401,43 @@ "-482e-b8b9-0e5e6887e411/mount" == watchdog.get_mountpoint_from_nfs_mounts(state_file_name, nfs_mounts) ) + + +def test_get_system_release_version_macos(mocker): + mocker.patch("watchdog.check_if_platform_is_mac", return_value=True) + platform_mock = mocker.patch("platform.platform", return_value=MACOS) + assert MACOS == watchdog.get_system_release_version() + utils.assert_called_once(platform_mock) + + +def test_get_system_release_version_linux_read_from_sys_release_path(mocker): + mocker.patch("watchdog.check_if_platform_is_mac", return_value=False) + open_mock = mocker.patch("builtins.open", mock_open(read_data=AL2)) + platform_mock = mocker.patch("platform.platform") + assert AL2 == watchdog.get_system_release_version() + utils.assert_not_called(platform_mock) + utils.assert_called_once(open_mock) + + [email protected](sys.version_info[1] < 7, "Not supported in python3.6 and below.") +def test_get_system_release_version_linux_read_from_os_release_path(mocker): + mocker.patch("watchdog.check_if_platform_is_mac", return_value=False) + mock = mock_open() + mock.side_effect = [ + FileNotFoundError, + mock_open(read_data="PRETTY_NAME=Amazon Linux release 2").return_value, + ] + open_mock = mocker.patch("builtins.open", mock) + platform_mock = mocker.patch("platform.platform") + assert AL2 == watchdog.get_system_release_version() + utils.assert_not_called(platform_mock) + utils.assert_called_n_times(open_mock, 2) + + +def test_get_system_release_version_linux_unknown(mocker): + mocker.patch("watchdog.check_if_platform_is_mac", return_value=False) + open_mock = mocker.patch("builtins.open", side_effect=FileNotFoundError) + platform_mock = mocker.patch("platform.platform") + assert watchdog.DEFAULT_UNKNOWN_VALUE == watchdog.get_system_release_version() + utils.assert_not_called(platform_mock) + utils.assert_called_n_times(open_mock, 2) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/efs-utils-1.33.4/test/watchdog_test/test_send_signal_to_stunnel_processes.py new/efs-utils-1.34.3/test/watchdog_test/test_send_signal_to_stunnel_processes.py --- old/efs-utils-1.33.4/test/watchdog_test/test_send_signal_to_stunnel_processes.py 2022-09-22 07:36:56.000000000 +0200 +++ new/efs-utils-1.34.3/test/watchdog_test/test_send_signal_to_stunnel_processes.py 2022-12-02 12:17:58.000000000 +0100 @@ -114,10 +114,12 @@ assert not os.path.exists(os.path.join(str(mount_dir), watchdog.STUNNEL_PID_FILE)) mock_log_debug = mocker.patch("logging.debug") - assert False == watchdog.is_mount_stunnel_proc_running(PID, STATE_FILE, str(tmpdir)) + assert True == watchdog.is_mount_stunnel_proc_running(PID, STATE_FILE, str(tmpdir)) - debug_log = mock_log_debug.call_args[0][0] - assert "Pid file of stunnel is already removed" in debug_log + assert "Pid file of stunnel does not exist" in str( + mock_log_debug.call_args_list[0][0] + ) + assert "is running with pid " in str(mock_log_debug.call_args_list[1][0]) def test_is_mount_stunnel_proc_running_pid_mismatch(mocker, tmpdir): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/efs-utils-1.33.4/test/watchdog_test/test_start_tls_tunnel.py new/efs-utils-1.34.3/test/watchdog_test/test_start_tls_tunnel.py --- old/efs-utils-1.33.4/test/watchdog_test/test_start_tls_tunnel.py 2022-09-22 07:36:56.000000000 +0200 +++ new/efs-utils-1.34.3/test/watchdog_test/test_start_tls_tunnel.py 2022-12-02 12:17:58.000000000 +0100 @@ -4,50 +4,158 @@ # for the specific language governing permissions and limitations under # the License. +import json +import os +import tempfile +import unittest from unittest.mock import MagicMock import pytest import watchdog +from .. import utils + PID = 1234 -def _mock_popen(mocker, returncode=0): +def _get_popen_mock(pid=PID): popen_mock = MagicMock() - popen_mock.communicate.return_value = ( - "stdout", - "stderr", - ) - popen_mock.pid = PID - popen_mock.returncode = returncode + popen_mock.pid = pid + return popen_mock + + +def _mock_popen(mocker): + return mocker.patch("subprocess.Popen", return_value=_get_popen_mock()) + - return mocker.patch("subprocess.Popen", return_value=popen_mock) +def _initiate_state_file(tmpdir, cmd=None): + state = { + "pid": PID - 1, + "cmd": cmd + if cmd + else [ + "/usr/bin/stunnel", + "/var/run/efs/stunnel-config.fs-deadbeef.mnt.21007", + ], + } + state_file = tempfile.mkstemp(prefix="state", dir=str(tmpdir))[1] + with open(state_file, "w") as f: + f.write(json.dumps(state)) + return state, state_file -def test_start_tls_tunnel(mocker): +def test_start_tls_tunnel(mocker, tmpdir): _mock_popen(mocker) mocker.patch("watchdog.is_pid_running", return_value=True) + state, state_file = _initiate_state_file(tmpdir) procs = [] - pid = watchdog.start_tls_tunnel(procs, "fs-deadbeef", "stunnel") + pid = watchdog.start_tls_tunnel(procs, state, str(tmpdir), state_file) assert PID == pid assert 1 == len(procs) -def test_start_tls_tunnel_fails(mocker, capsys): +def test_start_tls_tunnel_fails(mocker, capsys, tmpdir): _mock_popen(mocker) mocker.patch("watchdog.is_pid_running", return_value=False) + state, state_file = _initiate_state_file(tmpdir) procs = [] - with pytest.raises(SystemExit) as ex: - watchdog.start_tls_tunnel(procs, "fs-deadbeef", "stunnel") + watchdog.start_tls_tunnel(procs, state, str(tmpdir), state_file) assert 0 == len(procs) - assert 0 != ex.value.code out, err = capsys.readouterr() assert "Failed to initialize TLS tunnel" in err + + +# https://github.com/kubernetes-sigs/aws-efs-csi-driver/issues/812 The watchdog is trying to launch stunnel on AL2 for +# mounts using older version of efs-utils based on old state file command, while somehow the stunnel bin is removed +# after updating driver with new efs-utils(using stunnel5 bin). Watchdog should handle the case when the stunnel cannot +# be found, and fallback to use stunnel5 on AL2. +# +def _test_start_tls_tunnel_for_mount_via_older_version_of_efs_utils_on_amazon_linux_2_helper( + mocker, tmpdir, release_version +): + popen_mocker = mocker.patch( + "subprocess.Popen", side_effect=[FileNotFoundError, _get_popen_mock()] + ) + mocker.patch("watchdog.is_pid_running", return_value=True) + mocker.patch( + "watchdog.get_system_release_version", + return_value=release_version, + ) + mocker.patch("watchdog.find_command_path", return_value="/usr/sbin/stunnel5") + + state, state_file = _initiate_state_file(tmpdir) + procs = [] + pid = watchdog.start_tls_tunnel( + procs, state, str(tmpdir), state_file.split("/")[-1] + ) + assert PID == pid + assert 1 == len(procs) + utils.assert_called_n_times(popen_mocker, 2) + + with open(state_file) as f: + state = json.load(f) + + assert "/usr/sbin/stunnel5" == state["cmd"][0] + + +def test_start_tls_tunnel_for_mount_via_older_version_of_efs_utils_on_amazon_linux_2_with_release_id( + mocker, tmpdir +): + _test_start_tls_tunnel_for_mount_via_older_version_of_efs_utils_on_amazon_linux_2_helper( + mocker, tmpdir, watchdog.AMAZON_LINUX_2_RELEASE_ID + ) + + +def test_start_tls_tunnel_for_mount_via_older_version_of_efs_utils_on_amazon_linux_2_with_pretty_name( + mocker, tmpdir +): + _test_start_tls_tunnel_for_mount_via_older_version_of_efs_utils_on_amazon_linux_2_helper( + mocker, tmpdir, watchdog.AMAZON_LINUX_2_PRETTY_NAME + ) + + +# On ECS AL2, the stunnel is started in the given network namespace, i.e. the command to start stunnel is different. +# e.g. nsenter --net=/proc/1234/ns/net /usr/bin/stunnel /var/run/efs/stunnel-config.fs-deadbeef.12345 +# Need to make sure after we detect the stunnel is not found, we update the old mount to use stunnel5, and command is +# nsenter --net=/proc/1234/ns/net /usr/bin/stunnel5 /var/run/efs/stunnel-config.fs-deadbeef.12345 +# +def test_start_tls_tunnel_for_mount_via_older_version_of_efs_utils_on_ecs_amazon_linux_2( + mocker, tmpdir +): + _mock_popen(mocker) + mocker.patch("watchdog.is_pid_running", return_value=True) + mocker.patch( + "watchdog.get_system_release_version", + return_value=watchdog.AMAZON_LINUX_2_RELEASE_ID, + ) + mocker.patch("watchdog.find_command_path", return_value="/usr/sbin/stunnel5") + + namespace = "--net=/proc/1234/ns/net" + cmd = [ + "nsenter", + namespace, + "/usr/bin/stunnel", + "/var/run/efs/stunnel-config.fs-deadbeef.mnt.21007", + ] + state, state_file = _initiate_state_file(tmpdir, cmd=cmd) + procs = [] + pid = watchdog.start_tls_tunnel( + procs, state, str(tmpdir), state_file.split("/")[-1] + ) + assert PID == pid + assert 1 == len(procs) + + with open(state_file) as f: + state = json.load(f) + + assert " ".join(["nsenter", namespace, "/usr/sbin/stunnel5"]) in " ".join( + state["cmd"] + )
