Repository: ambari Updated Branches: refs/heads/trunk 4ecd3c762 -> 1d6ca13ae
AMBARI-15575: Stack Featurize Knox Service (jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1d6ca13a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1d6ca13a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1d6ca13a Branch: refs/heads/trunk Commit: 1d6ca13aec42e864235d9b5fa3047cbc1e9c0f7c Parents: 4ecd3c7 Author: Jayush Luniya <[email protected]> Authored: Thu Mar 31 13:57:06 2016 -0700 Committer: Jayush Luniya <[email protected]> Committed: Thu Mar 31 13:57:06 2016 -0700 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 2 + .../libraries/functions/stack_features.py | 15 +++++++- .../KNOX/0.5.0.2.2/package/scripts/knox.py | 6 ++- .../0.5.0.2.2/package/scripts/knox_gateway.py | 9 +++-- .../0.5.0.2.2/package/scripts/params_linux.py | 40 ++++++++++---------- .../0.5.0.2.2/package/scripts/status_params.py | 11 +++++- .../KNOX/0.5.0.2.2/package/scripts/upgrade.py | 8 ++-- .../HDP/2.0.6/properties/stack_features.json | 10 +++++ 8 files changed, 68 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1d6ca13a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index f444b65..1bfec09 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -61,3 +61,5 @@ class StackFeature: PIG_ON_TEZ = "pig_on_tez" RANGER_USERSYNC_NON_ROOT = "ranger_usersync_non_root" ACCUMULO_KERBEROS_USER_AUTH = "accumulo_kerberos_user_auth" + KNOX_VERSIONED_DATA_DIR = "knox_versioned_data_dir" + KNOX_SSO_TOPOLOGY = "knox_sso_topology" http://git-wip-us.apache.org/repos/asf/ambari/blob/1d6ca13a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py index ff3ec45..bfbf250 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py @@ -138,6 +138,16 @@ _DEFAULT_STACK_FEATURES = { "name": "accumulo_kerberos_user_auth", "description": "Accumulo Kerberos User Auth (AMBARI-10163)", "min_version": "2.3.0.0" + }, + { + "name": "knox_versioned_data_dir", + "description": "Use versioned data dir for Knox (AMBARI-13164)", + "min_version": "2.3.2.0" + }, + { + "name": "knox_sso_topology", + "description": "Knox SSO Topology support (AMBARI-13975)", + "min_version": "2.3.8.0" } ] } @@ -151,7 +161,10 @@ def check_stack_feature(stack_feature, stack_version): """ stack_features_config = default("/configurations/cluster-env/stack_features", None) data = _DEFAULT_STACK_FEATURES - + + if not stack_version: + return False + if stack_features_config: data = json.loads(stack_features_config) http://git-wip-us.apache.org/repos/asf/ambari/blob/1d6ca13a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox.py index df10136..fbd879f 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox.py @@ -31,6 +31,8 @@ from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from resource_management.core.logger import Logger +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions import StackFeature @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) def knox(): @@ -66,7 +68,7 @@ def knox(): content=InlineTemplate(params.admin_topology_template) ) - if Script.is_stack_greater_or_equal_to(params.version_formatted, "2.3.8.0"): + if params.version_formatted and check_stack_feature(StackFeature.KNOX_SSO_TOPOLOGY, params.version_formatted): File(os.path.join(params.knox_conf_dir, "topologies", "knoxsso.xml"), group=params.knox_group, owner=params.knox_user, @@ -123,7 +125,7 @@ def knox(): content=InlineTemplate(params.admin_topology_template) ) - if Script.is_stack_greater_or_equal_to(params.version_formatted, "2.3.8.0"): + if params.version_formatted and check_stack_feature(StackFeature.KNOX_SSO_TOPOLOGY, params.version_formatted): File(os.path.join(params.knox_conf_dir, "topologies", "knoxsso.xml"), group=params.knox_group, owner=params.knox_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/1d6ca13a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py index b72fc6f..aa5ce1f 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py @@ -25,7 +25,6 @@ from resource_management.libraries.functions import conf_select, tar_archive from resource_management.libraries.functions import stack_select from resource_management.libraries.functions.check_process_status import check_process_status from resource_management.libraries.functions import format -from resource_management.libraries.functions.version import compare_versions, format_stack_version from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import Direction @@ -47,11 +46,14 @@ import upgrade from knox import knox, update_knox_logfolder_permissions from knox_ldap import ldap from setup_ranger_knox import setup_ranger_knox +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions import StackFeature class KnoxGateway(Script): def get_stack_to_component(self): - return {"HDP": "knox-server"} + import params + return {params.stack_name: "knox-server"} def install(self, env): import params @@ -113,8 +115,7 @@ class KnoxGatewayDefault(KnoxGateway): def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) - if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0: - + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): absolute_backup_dir = None if params.upgrade_direction and params.upgrade_direction == Direction.UPGRADE: Logger.info("Backing up directories. Initial conf folder: %s" % os.path.realpath(params.knox_conf_dir)) http://git-wip-us.apache.org/repos/asf/ambari/blob/1d6ca13a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py index e3efe7c..79bdb3e 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py @@ -33,9 +33,12 @@ from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import conf_select from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions import StackFeature # server configurations config = Script.get_config() +stack_root = Script.get_stack_root() tmp_dir = Script.get_tmp_dir() stack_name = default("/hostLevelParams/stack_name", None) @@ -54,18 +57,16 @@ upgrade_from_version = default("/hostLevelParams/current_version", None) # server configurations # Default value used in HDP 2.3.0.0 and earlier. - knox_data_dir = '/var/lib/knox/data' # Important, it has to be strictly greater than 2.3.0.0!!! -if stack_name and stack_name.upper() == "HDP": - Logger.info(format("HDP version to use is {version_formatted}")) - if Script.is_stack_greater(version_formatted, "2.3.0.0"): - # This is the current version. In the case of a Rolling Upgrade, it will be the newer version. - # In the case of a Downgrade, it will be the version downgrading to. - # This is always going to be a symlink to /var/lib/knox/data_${version} - knox_data_dir = format('/usr/hdp/{version}/knox/data') - Logger.info(format("Detected HDP with stack version {version}, will use knox_data_dir = {knox_data_dir}")) +Logger.info(format("Stack version to use is {version_formatted}")) +if version_formatted and check_stack_feature(StackFeature.KNOX_VERSIONED_DATA_DIR, version_formatted): + # This is the current version. In the case of a Rolling Upgrade, it will be the newer version. + # In the case of a Downgrade, it will be the version downgrading to. + # This is always going to be a symlink to /var/lib/knox/data_${version} + knox_data_dir = format('{stack_root}/{version}/knox/data') + Logger.info(format("Detected stack with version {version}, will use knox_data_dir = {knox_data_dir}")) knox_master_secret_path = format('{knox_data_dir}/security/master') @@ -83,15 +84,14 @@ ldap_bin = '/usr/lib/knox/bin/ldap.sh' knox_client_bin = '/usr/lib/knox/bin/knoxcli.sh' # HDP 2.2+ parameters -if Script.is_stack_greater_or_equal("2.2"): - knox_bin = '/usr/hdp/current/knox-server/bin/gateway.sh' - knox_conf_dir = '/usr/hdp/current/knox-server/conf' - ldap_bin = '/usr/hdp/current/knox-server/bin/ldap.sh' - knox_client_bin = '/usr/hdp/current/knox-server/bin/knoxcli.sh' - - knox_master_secret_path = '/usr/hdp/current/knox-server/data/security/master' - knox_cert_store_path = '/usr/hdp/current/knox-server/data/security/keystores/gateway.jks' - knox_data_dir = '/usr/hdp/current/knox-server/data/' +if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): + knox_bin = format('{stack_root}/current/knox-server/bin/gateway.sh') + knox_conf_dir = format('{stack_root}/current/knox-server/conf') + ldap_bin = format('{stack_root}/current/knox-server/bin/ldap.sh') + knox_client_bin = format('{stack_root}/current/knox-server/bin/knoxcli.sh') + knox_master_secret_path = format('{stack_root}/current/knox-server/data/security/master') + knox_cert_store_path = format('{stack_root}/current/knox-server/data/security/keystores/gateway.jks') + knox_data_dir = format('{stack_root}/current/knox-server/data/') knox_group = default("/configurations/knox-env/knox_group", "knox") mode = 0644 @@ -211,7 +211,7 @@ if has_oozie: oozie_server_port = get_port_from_url(config['configurations']['oozie-site']['oozie.base.url']) # Knox managed properties -knox_managed_pid_symlink= "/usr/hdp/current/knox-server/pids" +knox_managed_pid_symlink= format('{stack_root}/current/knox-server/pids') # server configurations knox_master_secret = config['configurations']['knox-env']['knox_master_secret'] @@ -301,7 +301,7 @@ if has_ranger_admin: downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") - driver_curl_target = format("/usr/hdp/current/knox-server/ext/{jdbc_jar_name}") + driver_curl_target = format("{stack_root}/current/knox-server/ext/{jdbc_jar_name}") knox_ranger_plugin_config = { 'username': repo_config_username, http://git-wip-us.apache.org/repos/asf/ambari/blob/1d6ca13a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/status_params.py index b1a5ebc..b4e8831 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/status_params.py @@ -22,16 +22,23 @@ from resource_management.libraries.functions.default import default from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.script.script import Script from ambari_commons import OSCheck +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions import StackFeature + config = Script.get_config() +stack_root = Script.get_stack_root() +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) if OSCheck.is_windows_family(): knox_gateway_win_service_name = "gateway" knox_ldap_win_service_name = "ldap" else: knox_conf_dir = '/etc/knox/conf' - if Script.is_stack_greater_or_equal("2.2"): - knox_conf_dir = '/usr/hdp/current/knox-server/conf' + if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): + knox_conf_dir = format('{stack_root}/current/knox-server/conf') knox_pid_dir = config['configurations']['knox-env']['knox_pid_dir'] knox_pid_file = format("{knox_pid_dir}/gateway.pid") ldap_pid_file = format("{knox_pid_dir}/ldap.pid") http://git-wip-us.apache.org/repos/asf/ambari/blob/1d6ca13a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py index 8dd0cb2..63949f8 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py @@ -27,7 +27,8 @@ from resource_management.core.exceptions import Fail from resource_management.libraries.functions import tar_archive from resource_management.libraries.functions import format from resource_management.libraries.functions import Direction -from resource_management.libraries.functions.version import compare_versions,format_stack_version +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions import StackFeature BACKUP_TEMP_DIR = "knox-upgrade-backup" @@ -81,10 +82,9 @@ def _get_directory_mappings_during_upgrade(): # By default, use this for all stacks. knox_data_dir = '/var/lib/knox/data' - if params.stack_name and params.stack_name.upper() == "HDP" and \ - compare_versions(format_stack_version(params.upgrade_from_version), "2.3.0.0") > 0: + if params.upgrade_from_version and check_stack_feature(StackFeature.KNOX_VERSIONED_DATA_DIR, params.upgrade_from_version): # Use the version that is being upgraded from. - knox_data_dir = format('/usr/hdp/{upgrade_from_version}/knox/data') + knox_data_dir = format('{stack_root}/{upgrade_from_version}/knox/data') directories = {knox_data_dir: BACKUP_DATA_ARCHIVE, http://git-wip-us.apache.org/repos/asf/ambari/blob/1d6ca13a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index 6e4d672..7ac61eb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -112,6 +112,16 @@ "name": "accumulo_kerberos_user_auth", "description": "Accumulo Kerberos User Auth (AMBARI-10163)", "min_version": "2.3.0.0" + }, + { + "name": "knox_versioned_data_dir", + "description": "Use versioned data dir for Knox (AMBARI-13164)", + "min_version": "2.3.2.0" + }, + { + "name": "knox_sso_topology", + "description": "Knox SSO Topology support (AMBARI-13975)", + "min_version": "2.3.8.0" } ] }
