http://git-wip-us.apache.org/repos/asf/ambari/blob/ad09bb66/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/kms_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/kms_server.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/kms_server.py new file mode 100755 index 0000000..44d61da --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/kms_server.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.core.exceptions import Fail +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.functions import stack_select +from resource_management.libraries.script import Script +from resource_management.core.resources.system import Execute, File +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.libraries.functions.format import format +from resource_management.core.logger import Logger +from resource_management.core import shell +from resource_management.libraries.functions.default import default +from kms import kms, setup_kms_db, setup_java_patch, enable_kms_plugin, setup_kms_jce +from kms_service import kms_service +import upgrade + +class KmsServer(Script): + + def get_component_name(self): + return "ranger-kms" + + def install(self, env): + self.install_packages(env) + import params + env.set_params(params) + + setup_kms_db() + self.configure(env) + setup_java_patch() + + def stop(self, env, upgrade_type=None): + import params + + env.set_params(params) + kms_service(action = 'stop', upgrade_type=upgrade_type) + if params.stack_supports_pid: + File(params.ranger_kms_pid_file, + action = "delete" + ) + + def start(self, env, upgrade_type=None): + import params + + env.set_params(params) + self.configure(env) + enable_kms_plugin() + setup_kms_jce() + kms_service(action = 'start', upgrade_type=upgrade_type) + + def status(self, env): + import status_params + env.set_params(status_params) + + if status_params.stack_supports_pid: + check_process_status(status_params.ranger_kms_pid_file) + return + + cmd = 'ps -ef | grep proc_rangerkms | grep -v grep' + code, output = shell.call(cmd, timeout=20) + if code != 0: + Logger.debug('KMS process not running') + raise ComponentIsNotRunning() + pass + + def configure(self, env): + import params + + env.set_params(params) + kms() + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + upgrade.prestart(env, "ranger-kms") + kms(upgrade_type=upgrade_type) + setup_java_patch() + + def setup_ranger_kms_database(self, env): + import params + env.set_params(params) + + upgrade_stack = stack_select._get_upgrade_stack() + if upgrade_stack is None: + raise Fail('Unable to determine the stack and stack version') + + stack_version = upgrade_stack[1] + Logger.info(format('Setting Ranger KMS database schema, using version {stack_version}')) + setup_kms_db(stack_version=stack_version) + + def get_log_folder(self): + import params + return params.kms_log_dir + + def get_user(self): + import params + return params.kms_user + +if __name__ == "__main__": + KmsServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/ad09bb66/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/kms_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/kms_service.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/kms_service.py new file mode 100644 index 0000000..2ff48c3 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/kms_service.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.resources.system import Execute, File +from resource_management.core import shell +from resource_management.libraries.functions.format import format +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.core.logger import Logger +from resource_management.libraries.functions.show_logs import show_logs +from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING +from resource_management.libraries.functions.constants import Direction +import os + +def kms_service(action='start', upgrade_type=None): + import params + + env_dict = {'JAVA_HOME': params.java_home} + if params.db_flavor.lower() == 'sqla': + env_dict = {'JAVA_HOME': params.java_home, 'LD_LIBRARY_PATH': params.ld_library_path} + + if action == 'start': + no_op_test = format('ps -ef | grep proc_rangerkms | grep -v grep') + cmd = format('{kms_home}/ranger-kms start') + try: + Execute(cmd, not_if=no_op_test, environment=env_dict, user=format('{kms_user}')) + except: + show_logs(params.kms_log_dir, params.kms_user) + raise + elif action == 'stop': + if upgrade_type == UPGRADE_TYPE_NON_ROLLING and params.upgrade_direction == Direction.UPGRADE: + if os.path.isfile(format('{kms_home}/ranger-kms')): + File(format('{kms_home}/ranger-kms'), + owner=params.kms_user, + group = params.kms_group + ) + cmd = format('{kms_home}/ranger-kms stop') + try: + Execute(cmd, environment=env_dict, user=format('{kms_user}')) + except: + show_logs(params.kms_log_dir, params.kms_user) + raise http://git-wip-us.apache.org/repos/asf/ambari/blob/ad09bb66/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/params.py new file mode 100755 index 0000000..2445f2e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/params.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os +from resource_management.libraries.functions import conf_select +from resource_management.libraries.script import Script +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.stack_features import get_stack_feature_version +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.get_bare_principal import get_bare_principal +from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.functions.setup_ranger_plugin_xml import generate_ranger_service_config +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import get_kinit_path + +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() +stack_root = Script.get_stack_root() + +stack_name = default("/hostLevelParams/stack_name", None) +version = default("/commandParams/version", None) +upgrade_direction = default("/commandParams/upgrade_direction", None) + +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) + +# get the correct version to use for checking stack features +version_for_stack_feature_checks = get_stack_feature_version(config) + +stack_supports_config_versioning = check_stack_feature(StackFeature.CONFIG_VERSIONING, version_for_stack_feature_checks) +stack_support_kms_hsm = check_stack_feature(StackFeature.RANGER_KMS_HSM_SUPPORT, version_for_stack_feature_checks) +stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks) +stack_supports_pid = check_stack_feature(StackFeature.RANGER_KMS_PID_SUPPORT, version_for_stack_feature_checks) +stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) +stack_supports_ranger_kms_ssl = check_stack_feature(StackFeature.RANGER_KMS_SSL, version_for_stack_feature_checks) + +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +security_enabled = config['configurations']['cluster-env']['security_enabled'] + +if stack_supports_config_versioning: + kms_home = format('{stack_root}/current/ranger-kms') + kms_conf_dir = format('{stack_root}/current/ranger-kms/conf') + +kms_log_dir = default("/configurations/kms-env/kms_log_dir", "/var/log/ranger/kms") +java_home = config['hostLevelParams']['java_home'] +kms_user = default("/configurations/kms-env/kms_user", "kms") +kms_group = default("/configurations/kms-env/kms_group", "kms") + +ranger_kms_audit_log_maxfilesize = default('/configurations/kms-log4j/ranger_kms_audit_log_maxfilesize',256) +ranger_kms_audit_log_maxbackupindex = default('/configurations/kms-log4j/ranger_kms_audit_log_maxbackupindex',20) +ranger_kms_log_maxfilesize = default('/configurations/kms-log4j/ranger_kms_log_maxfilesize',256) +ranger_kms_log_maxbackupindex = default('/configurations/kms-log4j/ranger_kms_log_maxbackupindex',20) + +jdk_location = config['hostLevelParams']['jdk_location'] +kms_log4j = config['configurations']['kms-log4j']['content'] + +# ranger host +ranger_admin_hosts = config['clusterHostInfo']['ranger_admin_hosts'][0] +has_ranger_admin = len(ranger_admin_hosts) > 0 +kms_host = config['clusterHostInfo']['ranger_kms_server_hosts'][0] +kms_port = config['configurations']['kms-env']['kms_port'] + +create_db_user = config['configurations']['kms-env']['create_db_user'] + +#kms properties +db_flavor = (config['configurations']['kms-properties']['DB_FLAVOR']).lower() +db_host = config['configurations']['kms-properties']['db_host'] +db_name = config['configurations']['kms-properties']['db_name'] +db_user = config['configurations']['kms-properties']['db_user'] +db_password = unicode(config['configurations']['kms-properties']['db_password']) +kms_master_key_password = unicode(config['configurations']['kms-properties']['KMS_MASTER_KEY_PASSWD']) +credential_provider_path = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.credential.provider.path'] +jdbc_alias = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.credential.alias'] +masterkey_alias = config['configurations']['dbks-site']['ranger.ks.masterkey.credential.alias'] +repo_name = str(config['clusterName']) + '_kms' +repo_name_value = config['configurations']['ranger-kms-security']['ranger.plugin.kms.service.name'] +if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": + repo_name = repo_name_value +cred_lib_path = os.path.join(kms_home,"cred","lib","*") +cred_setup_prefix = (format('{kms_home}/ranger_credential_helper.py'), '-l', cred_lib_path) +credential_file = format('/etc/ranger/{repo_name}/cred.jceks') + +if has_ranger_admin: + policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] + if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'): + policymgr_mgr_url = policymgr_mgr_url.rstrip('/') + xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() + xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits') + xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') + xa_audit_db_password = '' + if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db: + xa_audit_db_password = config['configurations']['admin-properties']['audit_db_password'] + xa_db_host = config['configurations']['admin-properties']['db_host'] + + admin_uname = config['configurations']['ranger-env']['admin_username'] + admin_password = config['configurations']['ranger-env']['admin_password'] + ambari_ranger_admin = config['configurations']['ranger-env']['ranger_admin_username'] + ambari_ranger_password = config['configurations']['ranger-env']['ranger_admin_password'] + admin_uname_password = format("{admin_uname}:{admin_password}") + ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] + +default_connectors_map = { "mssql":"sqljdbc4.jar", + "mysql":"mysql-connector-java.jar", + "postgres":"postgresql-jdbc.jar", + "oracle":"ojdbc.jar", + "sqla":"sajdbc4.jar"} + +java_share_dir = '/usr/share/java' +jdbc_jar_name = None +previous_jdbc_jar_name = None +if db_flavor == 'mysql': + jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) + db_jdbc_url = format('jdbc:log4jdbc:mysql://{db_host}/{db_name}') + db_jdbc_driver = "com.mysql.jdbc.Driver" + jdbc_dialect = "org.eclipse.persistence.platform.database.MySQLPlatform" +elif db_flavor == 'oracle': + jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) + colon_count = db_host.count(':') + if colon_count == 2 or colon_count == 0: + db_jdbc_url = format('jdbc:oracle:thin:@{db_host}') + else: + db_jdbc_url = format('jdbc:oracle:thin:@//{db_host}') + db_jdbc_driver = "oracle.jdbc.OracleDriver" + jdbc_dialect = "org.eclipse.persistence.platform.database.OraclePlatform" +elif db_flavor == 'postgres': + jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) + db_jdbc_url = format('jdbc:postgresql://{db_host}/{db_name}') + db_jdbc_driver = "org.postgresql.Driver" + jdbc_dialect = "org.eclipse.persistence.platform.database.PostgreSQLPlatform" +elif db_flavor == 'mssql': + jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) + db_jdbc_url = format('jdbc:sqlserver://{db_host};databaseName={db_name}') + db_jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" + jdbc_dialect = "org.eclipse.persistence.platform.database.SQLServerPlatform" +elif db_flavor == 'sqla': + jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) + db_jdbc_url = format('jdbc:sqlanywhere:database={db_name};host={db_host}') + db_jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" + jdbc_dialect = "org.eclipse.persistence.platform.database.SQLAnywherePlatform" + +downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") + +driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") +driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}") +previous_jdbc_jar = format("{kms_home}/ews/webapp/lib/{previous_jdbc_jar_name}") +ews_lib_jar_path = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}") + +if db_flavor == 'sqla': + downloaded_custom_connector = format("{tmp_dir}/sqla-client-jdbc.tar.gz") + jar_path_in_archive = format("{tmp_dir}/sqla-client-jdbc/java/sajdbc4.jar") + libs_path_in_archive = format("{tmp_dir}/sqla-client-jdbc/native/lib64/*") + jdbc_libs_dir = format("{kms_home}/native/lib64") + ld_library_path = format("{jdbc_libs_dir}") + +if has_ranger_admin: + xa_previous_jdbc_jar_name = None + if stack_supports_ranger_audit_db: + if xa_audit_db_flavor == 'mysql': + jdbc_jar = default("/hostLevelParams/custom_mysql_jdbc_name", None) + xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) + audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') + jdbc_driver = "com.mysql.jdbc.Driver" + elif xa_audit_db_flavor == 'oracle': + jdbc_jar = default("/hostLevelParams/custom_oracle_jdbc_name", None) + xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) + colon_count = xa_db_host.count(':') + if colon_count == 2 or colon_count == 0: + audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}') + else: + audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}') + jdbc_driver = "oracle.jdbc.OracleDriver" + elif xa_audit_db_flavor == 'postgres': + jdbc_jar = default("/hostLevelParams/custom_postgres_jdbc_name", None) + xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) + audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}') + jdbc_driver = "org.postgresql.Driver" + elif xa_audit_db_flavor == 'mssql': + jdbc_jar = default("/hostLevelParams/custom_mssql_jdbc_name", None) + xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) + audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}') + jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" + elif xa_audit_db_flavor == 'sqla': + jdbc_jar = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) + xa_previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) + audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') + jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" + + downloaded_connector_path = format("{tmp_dir}/{jdbc_jar}") if stack_supports_ranger_audit_db else None + driver_source = format("{jdk_location}/{jdbc_jar}") if stack_supports_ranger_audit_db else None + driver_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar}") if stack_supports_ranger_audit_db else None + xa_previous_jdbc_jar = format("{kms_home}/ews/webapp/lib/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None + +repo_config_username = config['configurations']['kms-properties']['REPOSITORY_CONFIG_USERNAME'] +repo_config_password = unicode(config['configurations']['kms-properties']['REPOSITORY_CONFIG_PASSWORD']) + +kms_plugin_config = { + 'username' : repo_config_username, + 'password' : repo_config_password, + 'provider' : format('kms://http@{kms_host}:{kms_port}/kms') +} + +xa_audit_db_is_enabled = False +if stack_supports_ranger_audit_db: + xa_audit_db_is_enabled = config['configurations']['ranger-kms-audit']['xasecure.audit.destination.db'] +ssl_keystore_password = unicode(config['configurations']['ranger-kms-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) +ssl_truststore_password = unicode(config['configurations']['ranger-kms-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) + +#For SQLA explicitly disable audit to DB for Ranger +if xa_audit_db_flavor == 'sqla': + xa_audit_db_is_enabled = False + +current_host = config['hostname'] +ranger_kms_hosts = config['clusterHostInfo']['ranger_kms_server_hosts'] +if current_host in ranger_kms_hosts: + kms_host = current_host + +check_db_connection_jar_name = "DBConnectionVerification.jar" +check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}") +ranger_kms_jdbc_connection_url = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.url'] +ranger_kms_jdbc_driver = config['configurations']['dbks-site']['ranger.ks.jpa.jdbc.driver'] + +jce_name = default("/hostLevelParams/jce_name", None) +jce_source_dir = format('{tmp_dir}/jce_dir') + +#kms hsm support +enable_kms_hsm = default("/configurations/dbks-site/ranger.ks.hsm.enabled", False) +hms_partition_alias = default("/configurations/dbks-site/ranger.ks.hsm.partition.password.alias", "ranger.kms.hsm.partition.password") +hms_partition_passwd = default("/configurations/kms-env/hsm_partition_password", None) + +# kms kerberos from stack 2.5 onward +rangerkms_bare_principal = 'rangerkms' + +if stack_supports_ranger_kerberos: + if security_enabled: + rangerkms_principal = config['configurations']['dbks-site']['ranger.ks.kerberos.principal'] + rangerkms_keytab = config['configurations']['dbks-site']['ranger.ks.kerberos.keytab'] + if not is_empty(rangerkms_principal) and rangerkms_principal != '': + rangerkms_bare_principal = get_bare_principal(rangerkms_principal) + rangerkms_principal = rangerkms_principal.replace('_HOST', kms_host.lower()) + kms_plugin_config['policy.download.auth.users'] = format('keyadmin,{rangerkms_bare_principal}') + +custom_ranger_service_config = generate_ranger_service_config(config['configurations']['kms-properties']) +if len(custom_ranger_service_config) > 0: + kms_plugin_config.update(custom_ranger_service_config) + +kms_ranger_plugin_repo = { + 'isEnabled' : 'true', + 'configs' : kms_plugin_config, + 'description' : 'kms repo', + 'name' : repo_name, + 'type' : 'kms' +} + +# ranger kms pid +user_group = config['configurations']['cluster-env']['user_group'] +ranger_kms_pid_dir = default("/configurations/kms-env/ranger_kms_pid_dir", "/var/run/ranger_kms") +ranger_kms_pid_file = format('{ranger_kms_pid_dir}/rangerkms.pid') + +if security_enabled: + spengo_keytab = config['configurations']['kms-site']['hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab'] + spnego_principal = config['configurations']['kms-site']['hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal'] + spnego_principal = spnego_principal.replace('_HOST', current_host.lower()) + +plugin_audit_password_property = 'xasecure.audit.destination.db.password' +kms_plugin_password_properties = ['xasecure.policymgr.clientssl.keystore.password', 'xasecure.policymgr.clientssl.truststore.password'] +dbks_site_password_properties = ['ranger.db.encrypt.key.password', 'ranger.ks.jpa.jdbc.password', 'ranger.ks.hsm.partition.password'] +ranger_kms_site_password_properties = ['ranger.service.https.attrib.keystore.pass'] +ranger_kms_cred_ssl_path = config['configurations']['ranger-kms-site']['ranger.credential.provider.path'] +ranger_kms_ssl_keystore_alias = config['configurations']['ranger-kms-site']['ranger.service.https.attrib.keystore.credential.alias'] +ranger_kms_ssl_passwd = config['configurations']['ranger-kms-site']['ranger.service.https.attrib.keystore.pass'] +ranger_kms_ssl_enabled = config['configurations']['ranger-kms-site']['ranger.service.https.attrib.ssl.enabled'] + +xa_audit_hdfs_is_enabled = default("/configurations/ranger-kms-audit/xasecure.audit.destination.hdfs", False) +namenode_host = default("/clusterHostInfo/namenode_host", []) + +# need this to capture cluster name from where ranger kms plugin is enabled +cluster_name = config['clusterName'] + +has_namenode = len(namenode_host) > 0 + +hdfs_user = default("/configurations/hadoop-env/hdfs_user", None) +hdfs_user_keytab = default("/configurations/hadoop-env/hdfs_user_keytab", None) +hdfs_principal_name = default("/configurations/hadoop-env/hdfs_principal_name", None) +default_fs = default("/configurations/core-site/fs.defaultFS", None) +hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) + +import functools +# create partial functions with common arguments for every HdfsResource call +# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs +) + +local_component_list = default("/localComponents", []) +has_hdfs_client_on_node = 'HDFS_CLIENT' in local_component_list \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ad09bb66/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/service_check.py new file mode 100644 index 0000000..84e4e73 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/service_check.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script +from resource_management.core.logger import Logger +from resource_management.core import shell +from resource_management.core.exceptions import ComponentIsNotRunning + + +class KmsServiceCheck(Script): + def service_check(self, env): + import params + + env.set_params(params) + cmd = 'ps -ef | grep proc_rangerkms | grep -v grep' + code, output = shell.call(cmd, timeout=20) + if code == 0: + Logger.info('KMS process up and running') + else: + Logger.debug('KMS process not running') + raise ComponentIsNotRunning() + +if __name__ == "__main__": + KmsServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/ad09bb66/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/status_params.py new file mode 100644 index 0000000..34d0082 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/status_params.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions import StackFeature + +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +stack_name = default("/hostLevelParams/stack_name", None) +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) +stack_supports_pid = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KMS_PID_SUPPORT, stack_version_formatted) +ranger_kms_pid_dir = default("/configurations/kms-env/ranger_kms_pid_dir", "/var/run/ranger_kms") +ranger_kms_pid_file = format('{ranger_kms_pid_dir}/rangerkms.pid') \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ad09bb66/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/upgrade.py new file mode 100644 index 0000000..8478bb8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/scripts/upgrade.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.format import format + +def prestart(env, stack_component): + import params + + if params.version and params.stack_supports_config_versioning: + conf_select.select(params.stack_name, stack_component, params.version) + stack_select.select(stack_component, params.version) http://git-wip-us.apache.org/repos/asf/ambari/blob/ad09bb66/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/templates/input.config-ranger-kms.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/templates/input.config-ranger-kms.json.j2 b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/templates/input.config-ranger-kms.json.j2 new file mode 100644 index 0000000..306fade --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/package/templates/input.config-ranger-kms.json.j2 @@ -0,0 +1,48 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"ranger_kms", + "rowtype":"service", + "path":"{{default('/configurations/kms-env/kms_log_dir', '/var/log/ranger/kms')}}/kms.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_kms" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ad09bb66/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/role_command_order.json b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/role_command_order.json new file mode 100644 index 0000000..7ddab41 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/role_command_order.json @@ -0,0 +1,7 @@ +{ + "general_deps" : { + "_comment" : "dependencies for RANGER-KMS", + "RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START", "NAMENODE-START"], + "RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/ad09bb66/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/themes/theme_version_1.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/themes/theme_version_1.json b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/themes/theme_version_1.json new file mode 100644 index 0000000..c08a56c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/themes/theme_version_1.json @@ -0,0 +1,303 @@ +{ + "name": "default", + "description": "Default theme for Ranger KMS service", + "configuration": { + "layouts": [ + { + "name": "default", + "tabs": [ + { + "name": "db_settings", + "display-name": "Settings", + "layout": { + "tab-columns": "2", + "tab-rows": "2", + "sections": [ + { + "name": "section-db-settings", + "display-name": "", + "row-index": "0", + "column-index": "0", + "row-span": "4", + "column-span": "2", + "section-columns": "2", + "section-rows": "4", + "subsections": [ + { + "name": "subsection-kms-db-row1-col1", + "display-name": "Ranger KMS DB", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + }, + { + "name": "subsection-kms-db-row1-col2", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1" + }, + { + "name": "subsection-kms-create-db-user-row2-col", + "display-name": "Setup Database and Database User", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "2" + }, + { + "name": "subsection-kms-db-root-user-row3-col1", + "display-name": "Ranger KMS Root DB", + "row-index": "2", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "depends-on": [ + { + "configs":[ + "kms-env/create_db_user" + ], + "if": "${kms-env/create_db_user}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "name": "subsection-kms-db-root-user-row3-col2", + "row-index": "2", + "column-index": "1", + "row-span": "1", + "column-span": "1", + "depends-on": [ + { + "configs":[ + "kms-env/create_db_user" + ], + "if": "${kms-env/create_db_user}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "name": "subsection-kms-master-row4-col", + "display-name": "KMS Master Secret Password", + "row-index": "3", + "column-index": "0", + "row-span": "1", + "column-span": "2" + } + ] + } + ] + } + } + ] + } + ], + "placement": { + "configuration-layout": "default", + "configs": [ + { + "config": "kms-properties/DB_FLAVOR", + "subsection-name": "subsection-kms-db-row1-col1" + }, + { + "config": "kms-properties/db_name", + "subsection-name": "subsection-kms-db-row1-col1" + }, + { + "config": "dbks-site/ranger.ks.jpa.jdbc.url", + "subsection-name": "subsection-kms-db-row1-col1" + }, + { + "config": "kms-properties/db_user", + "subsection-name": "subsection-kms-db-row1-col1" + }, + { + "config": "kms-properties/db_host", + "subsection-name": "subsection-kms-db-row1-col2" + }, + { + "config": "kms-properties/SQL_CONNECTOR_JAR", + "subsection-name": "subsection-kms-db-row1-col2", + "depends-on" : [ + { + "configs":[ + "kms-properties/DB_FLAVOR" + ], + "if": "${kms-properties/DB_FLAVOR} === SQLA", + "then": { + "property_value_attributes": { + "visible": false + } + }, + "else": { + "property_value_attributes": { + "visible": true + } + } + } + ] + }, + { + "config": "dbks-site/ranger.ks.jpa.jdbc.driver", + "subsection-name": "subsection-kms-db-row1-col2" + }, + { + "config": "kms-properties/db_password", + "subsection-name": "subsection-kms-db-row1-col2" + }, + { + "config": "kms-properties/db_root_user", + "subsection-name": "subsection-kms-db-root-user-row3-col1" + }, + { + "config": "kms-properties/db_root_password", + "subsection-name": "subsection-kms-db-root-user-row3-col2" + }, + { + "config": "kms-properties/KMS_MASTER_KEY_PASSWD", + "subsection-name": "subsection-kms-master-row4-col" + }, + { + "config" : "kms-env/create_db_user", + "subsection-name": "subsection-kms-create-db-user-row2-col" + }, + { + "config": "kms-env/test_db_kms_connection", + "subsection-name": "subsection-kms-create-db-user-row2-col", + "property_value_attributes": { + "ui_only_property": true + }, + "depends-on": [ + { + "configs":[ + "kms-env/create_db_user" + ], + "if": "${kms-env/create_db_user}", + "then": { + "property_value_attributes": { + "visible": false + } + }, + "else": { + "property_value_attributes": { + "visible": true + } + } + } + ] + } + ] + }, + "widgets": [ + { + "config": "kms-properties/DB_FLAVOR", + "widget": { + "type": "combo" + } + }, + { + "config": "kms-properties/db_user", + "widget": { + "type": "text-field" + } + }, + { + "config": "kms-properties/db_name", + "widget": { + "type": "text-field" + } + }, + { + "config": "kms-properties/SQL_CONNECTOR_JAR", + "widget": { + "type": "text-field" + } + }, + { + "config": "kms-properties/db_root_user", + "widget": { + "type": "text-field" + } + }, + { + "config": "kms-properties/db_host", + "widget": { + "type": "text-field" + } + }, + { + "config": "kms-properties/db_password", + "widget": { + "type": "password" + } + }, + { + "config": "kms-properties/db_root_password", + "widget": { + "type": "password" + } + }, + { + "config": "kms-properties/KMS_MASTER_KEY_PASSWD", + "widget": { + "type": "password" + } + }, + { + "config": "kms-env/create_db_user", + "widget": { + "type": "toggle" + } + }, + { + "config": "kms-env/test_db_kms_connection", + "widget": { + "type": "test-db-connection", + "display-name": "Test Connection", + "required-properties": { + "jdbc.driver.class": "dbks-site/ranger.ks.jpa.jdbc.driver", + "jdbc.driver.url": "dbks-site/ranger.ks.jpa.jdbc.url", + "db.connection.source.host": "ranger_kms-site/ranger_kms_server_hosts", + "db.type": "kms-properties/DB_FLAVOR", + "db.connection.destination.host": "kms-properties/db_host", + "db.connection.user": "kms-properties/db_user", + "db.connection.password": "kms-properties/db_password" + } + } + }, + { + "config": "dbks-site/ranger.ks.jpa.jdbc.driver", + "widget" : { + "type": "text-field" + } + }, + { + "config": "dbks-site/ranger.ks.jpa.jdbc.url", + "widget": { + "type": "text-field" + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ad09bb66/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/themes/theme_version_2.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/themes/theme_version_2.json b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/themes/theme_version_2.json new file mode 100644 index 0000000..be50dad --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.3.0/themes/theme_version_2.json @@ -0,0 +1,124 @@ +{ + "configuration": { + "layouts": [ + { + "name": "default", + "tabs": [ + { + "name": "kms_hsm", + "display-name": "KMS HSM", + "layout": { + "tab-columns": "1", + "tab-rows": "1", + "sections": [ + { + "name": "section-kms-hms", + "display-name": "", + "row-index": "0", + "column-index": "0", + "row-span": "2", + "column-span": "1", + "section-columns": "1", + "section-rows": "2", + "subsections": [ + { + "name": "subsection-kms-hsm-row1-col1", + "display-name": "Ranger KMS HSM Enabled", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + }, + { + "name": "subsection-kms-hsm-row2-col1", + "display-name": "Configuration Settings", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "depends-on": [ + { + "configs": [ + "dbks-site/ranger.ks.hsm.enabled" + ], + "if": "${dbks-site/ranger.ks.hsm.enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + } + ] + } + ] + } + } + ] + } + ], + "placement": { + "configuration-layout": "default", + "configs": [ + { + "config": "dbks-site/ranger.ks.hsm.enabled", + "subsection-name": "subsection-kms-hsm-row1-col1" + }, + { + "config": "dbks-site/ranger.ks.hsm.type", + "subsection-name": "subsection-kms-hsm-row2-col1" + }, + { + "config": "dbks-site/ranger.ks.hsm.partition.name", + "subsection-name": "subsection-kms-hsm-row2-col1" + }, + { + "config": "dbks-site/ranger.ks.hsm.partition.password.alias", + "subsection-name": "subsection-kms-hsm-row2-col1" + }, + { + "config": "kms-env/hsm_partition_password", + "subsection-name": "subsection-kms-hsm-row2-col1" + } + ] + }, + "widgets": [ + { + "config": "dbks-site/ranger.ks.hsm.enabled", + "widget": { + "type": "toggle" + } + }, + { + "config": "dbks-site/ranger.ks.hsm.type", + "widget": { + "type": "combo" + } + }, + { + "config": "dbks-site/ranger.ks.hsm.partition.name", + "widget": { + "type": "text-field" + } + }, + { + "config": "dbks-site/ranger.ks.hsm.partition.password.alias", + "widget": { + "type": "text-field" + } + }, + { + "config": "kms-env/hsm_partition_password", + "widget": { + "type": "password" + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ad09bb66/ambari-server/src/main/resources/stacks/HDP/3.0/services/RANGER_KMS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/RANGER_KMS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/RANGER_KMS/metainfo.xml new file mode 100644 index 0000000..3375d90 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/RANGER_KMS/metainfo.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>RANGER_KMS</name> + <version>0.5.0.3.0</version> + <extends>common-services/RANGER_KMS/0.5.0.3.0</extends> + </service> + </services> +</metainfo>
