http://git-wip-us.apache.org/repos/asf/ambari/blob/8f051fc5/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/namenode_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/namenode_upgrade.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/namenode_upgrade.py new file mode 100644 index 0000000..f683dcc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/namenode_upgrade.py @@ -0,0 +1,322 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import re +import os + +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import File +from resource_management.core import shell +from resource_management.core.shell import as_user +from resource_management.core.exceptions import Fail +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions import get_unique_id_and_date +from resource_management.libraries.functions import Direction, SafeMode +from utils import get_dfsadmin_base_command + +from namenode_ha_state import NamenodeHAState + + +safemode_to_instruction = {SafeMode.ON: "enter", + SafeMode.OFF: "leave"} + +NAMENODE_UPGRADE_IN_PROGRESS_MARKER_FILE = "namenode-upgrade-in-progress" + +def prepare_upgrade_check_for_previous_dir(): + """ + During a NonRolling (aka Express Upgrade), preparing the NameNode requires backing up some data. + Check that there is no "previous" folder inside the NameNode Name Dir. + """ + import params + + if params.dfs_ha_enabled: + namenode_ha = NamenodeHAState() + if namenode_ha.is_active(params.hostname): + Logger.info("NameNode High Availability is enabled and this is the Active NameNode.") + + problematic_previous_namenode_dirs = set() + nn_name_dirs = params.dfs_name_dir.split(',') + for nn_dir in nn_name_dirs: + if os.path.isdir(nn_dir): + # Check for a previous folder, which is not allowed. + previous_dir = os.path.join(nn_dir, "previous") + if os.path.isdir(previous_dir): + problematic_previous_namenode_dirs.add(previous_dir) + + if len(problematic_previous_namenode_dirs) > 0: + message = 'WARNING. The following NameNode Name Dir(s) have a "previous" folder from an older version.\n' \ + 'Please back it up first, and then delete it, OR Finalize (E.g., "hdfs dfsadmin -finalizeUpgrade").\n' \ + 'NameNode Name Dir(s): {0}\n' \ + '***** Then, retry this step. *****'.format(", ".join(problematic_previous_namenode_dirs)) + Logger.error(message) + raise Fail(message) + +def prepare_upgrade_enter_safe_mode(hdfs_binary): + """ + During a NonRolling (aka Express Upgrade), preparing the NameNode requires first entering Safemode. + :param hdfs_binary: name/path of the HDFS binary to use + """ + import params + + dfsadmin_base_command = get_dfsadmin_base_command(hdfs_binary) + safe_mode_enter_cmd = dfsadmin_base_command + " -safemode enter" + try: + # Safe to call if already in Safe Mode + desired_state = SafeMode.ON + safemode_transition_successful, original_state = reach_safemode_state(params.hdfs_user, desired_state, params.dfs_ha_enabled, hdfs_binary) + Logger.info("Transition successful: {0}, original state: {1}".format(str(safemode_transition_successful), str(original_state))) + if not safemode_transition_successful: + raise Fail("Could not transition to safemode state %s. Please check logs to make sure namenode is up." % str(desired_state)) + except Exception, e: + message = "Could not enter safemode. Error: {0}. As the HDFS user, call this command: {1}".format(str(e), safe_mode_enter_cmd) + Logger.error(message) + raise Fail(message) + +def prepare_upgrade_save_namespace(hdfs_binary): + """ + During a NonRolling (aka Express Upgrade), preparing the NameNode requires saving the namespace. + :param hdfs_binary: name/path of the HDFS binary to use + """ + import params + + dfsadmin_base_command = get_dfsadmin_base_command(hdfs_binary) + save_namespace_cmd = dfsadmin_base_command + " -saveNamespace" + try: + Logger.info("Checkpoint the current namespace.") + as_user(save_namespace_cmd, params.hdfs_user, env={'PATH': params.hadoop_bin_dir}) + except Exception, e: + message = format("Could not save the NameSpace. As the HDFS user, call this command: {save_namespace_cmd}") + Logger.error(message) + raise Fail(message) + +def prepare_upgrade_backup_namenode_dir(): + """ + During a NonRolling (aka Express Upgrade), preparing the NameNode requires backing up the NameNode Name Dirs. + """ + import params + + i = 0 + failed_paths = [] + nn_name_dirs = params.dfs_name_dir.split(',') + backup_destination_root_dir = "{0}/{1}".format(params.namenode_backup_dir, params.stack_version_unformatted) + if len(nn_name_dirs) > 0: + Logger.info("Backup the NameNode name directory's CURRENT folder.") + for nn_dir in nn_name_dirs: + i += 1 + namenode_current_image = os.path.join(nn_dir, "current") + unique = get_unique_id_and_date() + "_" + str(i) + # Note that /tmp may not be writeable. + backup_current_folder = "{0}/namenode_{1}/".format(backup_destination_root_dir, unique) + + if os.path.isdir(namenode_current_image) and not os.path.isdir(backup_current_folder): + try: + os.makedirs(backup_current_folder) + Execute(('cp', '-ar', namenode_current_image, backup_current_folder), + sudo=True + ) + except Exception, e: + failed_paths.append(namenode_current_image) + if len(failed_paths) > 0: + Logger.error("Could not backup the NameNode Name Dir(s) to {0}, make sure that the destination path is " + "writeable and copy the directories on your own. Directories: {1}".format(backup_destination_root_dir, + ", ".join(failed_paths))) + +def prepare_upgrade_finalize_previous_upgrades(hdfs_binary): + """ + During a NonRolling (aka Express Upgrade), preparing the NameNode requires Finalizing any upgrades that are in progress. + :param hdfs_binary: name/path of the HDFS binary to use + """ + import params + + dfsadmin_base_command = get_dfsadmin_base_command(hdfs_binary) + finalize_command = dfsadmin_base_command + " -rollingUpgrade finalize" + try: + Logger.info("Attempt to Finalize if there are any in-progress upgrades. " + "This will return 255 if no upgrades are in progress.") + code, out = shell.checked_call(finalize_command, logoutput=True, user=params.hdfs_user) + if out: + expected_substring = "there is no rolling upgrade in progress" + if expected_substring not in out.lower(): + Logger.warning('Finalize command did not contain substring: %s' % expected_substring) + else: + Logger.warning("Finalize command did not return any output.") + except Exception, e: + Logger.warning("Ensure no upgrades are in progress.") + +def reach_safemode_state(user, safemode_state, in_ha, hdfs_binary): + """ + Enter or leave safemode for the Namenode. + :param user: user to perform action as + :param safemode_state: Desired state of ON or OFF + :param in_ha: bool indicating if Namenode High Availability is enabled + :param hdfs_binary: name/path of the HDFS binary to use + :return: Returns a tuple of (transition success, original state). If no change is needed, the indicator of + success will be True + """ + Logger.info("Prepare to transition into safemode state %s" % safemode_state) + import params + original_state = SafeMode.UNKNOWN + + dfsadmin_base_command = get_dfsadmin_base_command(hdfs_binary) + safemode_base_command = dfsadmin_base_command + " -safemode " + safemode_check_cmd = safemode_base_command + " get" + + grep_pattern = format("Safe mode is {safemode_state}") + safemode_check_with_grep = format("{safemode_check_cmd} | grep '{grep_pattern}'") + + code, out = shell.call(safemode_check_cmd, user=user, logoutput=True) + Logger.info("Command: %s\nCode: %d." % (safemode_check_cmd, code)) + if code == 0 and out is not None: + Logger.info(out) + re_pattern = r"Safe mode is (\S*)" + Logger.info("Pattern to search: {0}".format(re_pattern)) + m = re.search(re_pattern, out, re.IGNORECASE) + if m and len(m.groups()) >= 1: + original_state = m.group(1).upper() + + if original_state == safemode_state: + return (True, original_state) + else: + # Make a transition + command = safemode_base_command + safemode_to_instruction[safemode_state] + Execute(command, + user=user, + logoutput=True, + path=[params.hadoop_bin_dir]) + + code, out = shell.call(safemode_check_with_grep, user=user) + Logger.info("Command: %s\nCode: %d. Out: %s" % (safemode_check_with_grep, code, out)) + if code == 0: + return (True, original_state) + return (False, original_state) + + +def prepare_rolling_upgrade(hdfs_binary): + """ + This can be called during either Rolling Upgrade or Express Upgrade (aka nonrolling) + + Rolling Upgrade for HDFS Namenode requires the following. + 0. Namenode must be up + 1. If HA: leave safemode if the safemode status is not OFF + 2. Execute a rolling upgrade "prepare" + 3. Execute a rolling upgrade "query" + :param hdfs_binary: name/path of the HDFS binary to use + """ + import params + + if not params.upgrade_direction or params.upgrade_direction not in [Direction.UPGRADE, Direction.DOWNGRADE]: + raise Fail("Could not retrieve upgrade direction: %s" % str(params.upgrade_direction)) + Logger.info(format("Performing a(n) {params.upgrade_direction} of HDFS")) + + if params.security_enabled: + kinit_command = format("{params.kinit_path_local} -kt {params.hdfs_user_keytab} {params.hdfs_principal_name}") + Execute(kinit_command, user=params.hdfs_user, logoutput=True) + + if params.upgrade_direction == Direction.UPGRADE: + if params.dfs_ha_enabled: + Logger.info('High Availability is enabled, must leave safemode before calling "-rollingUpgrade prepare"') + desired_state = SafeMode.OFF + safemode_transition_successful, original_state = reach_safemode_state(params.hdfs_user, desired_state, True, hdfs_binary) + if not safemode_transition_successful: + raise Fail("Could not transition to safemode state %s. Please check logs to make sure namenode is up." % str(desired_state)) + + dfsadmin_base_command = get_dfsadmin_base_command(hdfs_binary) + prepare = dfsadmin_base_command + " -rollingUpgrade prepare" + query = dfsadmin_base_command + " -rollingUpgrade query" + Execute(prepare, + user=params.hdfs_user, + logoutput=True) + Execute(query, + user=params.hdfs_user, + logoutput=True) + +def finalize_upgrade(upgrade_type, hdfs_binary): + """ + Finalize the Namenode upgrade, at which point it cannot be downgraded. + :param upgrade_type rolling or nonrolling + :param hdfs_binary: name/path of the HDFS binary to use + """ + Logger.info("Executing Rolling Upgrade finalize") + import params + + if params.security_enabled: + kinit_command = format("{params.kinit_path_local} -kt {params.hdfs_user_keytab} {params.hdfs_principal_name}") + Execute(kinit_command, user=params.hdfs_user, logoutput=True) + + dfsadmin_base_command = get_dfsadmin_base_command(hdfs_binary) + finalize_cmd = dfsadmin_base_command + " -rollingUpgrade finalize" + query_cmd = dfsadmin_base_command + " -rollingUpgrade query" + + Execute(query_cmd, + user=params.hdfs_user, + logoutput=True) + Execute(finalize_cmd, + user=params.hdfs_user, + logoutput=True) + Execute(query_cmd, + user=params.hdfs_user, + logoutput=True) + + # upgrade is finalized; remove the upgrade marker + delete_upgrade_marker() + + +def get_upgrade_in_progress_marker(): + """ + Gets the full path of the file which indicates that NameNode has begun its stack upgrade. + :return: + """ + from resource_management.libraries.script.script import Script + return os.path.join(Script.get_tmp_dir(), NAMENODE_UPGRADE_IN_PROGRESS_MARKER_FILE) + + +def create_upgrade_marker(): + """ + Creates the marker file indicating that NameNode has begun participating in a stack upgrade. + If the file already exists, nothing will be done. This will silently log exceptions on failure. + :return: + """ + # create the marker file which indicates + try: + namenode_upgrade_in_progress_marker = get_upgrade_in_progress_marker() + if not os.path.isfile(namenode_upgrade_in_progress_marker): + File(namenode_upgrade_in_progress_marker) + except: + Logger.warning("Unable to create NameNode upgrade marker file {0}".format(namenode_upgrade_in_progress_marker)) + + +def delete_upgrade_marker(): + """ + Removes the marker file indicating that NameNode has begun participating in a stack upgrade. + If the file does not exist, then nothing will be done. + Failure to remove this file could cause problems with restarts in the future. That's why + checking to see if there is a suspended upgrade is also advised. This function will raise + an exception if the file can't be removed. + :return: + """ + # create the marker file which indicates + try: + namenode_upgrade_in_progress_marker = get_upgrade_in_progress_marker() + if os.path.isfile(namenode_upgrade_in_progress_marker): + File(namenode_upgrade_in_progress_marker, action='delete') + except: + error_message = "Unable to remove NameNode upgrade marker file {0}".format(namenode_upgrade_in_progress_marker) + Logger.error(error_message) + raise Fail(error_message) +
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f051fc5/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/nfsgateway.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/nfsgateway.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/nfsgateway.py new file mode 100644 index 0000000..770df59 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/nfsgateway.py @@ -0,0 +1,147 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script import Script +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.functions.security_commons import build_expectations, \ + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML +from hdfs_nfsgateway import nfsgateway +from hdfs import hdfs +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature + + +class NFSGateway(Script): + + def get_component_name(self): + return "hadoop-hdfs-nfs3" + + def install(self, env): + import params + + env.set_params(params) + + self.install_packages(env) + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + if params.stack_version_formatted and check_stack_feature(StackFeature.NFS, params.stack_version_formatted): + conf_select.select(params.stack_name, "hadoop", params.version) + stack_select.select("hadoop-hdfs-nfs3", params.version) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + + self.configure(env) + nfsgateway(action="start") + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + nfsgateway(action="stop") + + def configure(self, env): + import params + + env.set_params(params) + hdfs() + nfsgateway(action="configure") + + def status(self, env): + import status_params + + env.set_params(status_params) + + check_process_status(status_params.nfsgateway_pid_file) + + def security_status(self, env): + import status_params + + env.set_params(status_params) + props_value_check = {"hadoop.security.authentication": "kerberos", + "hadoop.security.authorization": "true"} + props_empty_check = ["hadoop.security.auth_to_local"] + props_read_check = None + core_site_expectations = build_expectations('core-site', props_value_check, props_empty_check, + props_read_check) + props_value_check = None + props_empty_check = ['nfs.keytab.file', + 'nfs.kerberos.principal'] + props_read_check = ['nfs.keytab.file'] + hdfs_site_expectations = build_expectations('hdfs-site', props_value_check, props_empty_check, + props_read_check) + + hdfs_expectations = {} + hdfs_expectations.update(core_site_expectations) + hdfs_expectations.update(hdfs_site_expectations) + + security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, + {'core-site.xml': FILE_TYPE_XML, + 'hdfs-site.xml': FILE_TYPE_XML}) + if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \ + security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos': + result_issues = validate_security_config_properties(security_params, hdfs_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ('hdfs-site' not in security_params or + 'nfs.keytab.file' not in security_params['hdfs-site'] or + 'nfs.kerberos.principal' not in security_params['hdfs-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.hdfs_user, + security_params['hdfs-site']['nfs.keytab.file'], + security_params['hdfs-site'][ + 'nfs.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) + else: + self.put_structured_out({"securityState": "UNSECURED"}) + + def get_log_folder(self): + import params + return params.hdfs_log_dir + + def get_user(self): + import params + return params.hdfs_user + +if __name__ == "__main__": + NFSGateway().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/8f051fc5/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params.py new file mode 100644 index 0000000..aa072dc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params.py @@ -0,0 +1,30 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from ambari_commons import OSCheck +from resource_management.libraries.functions.default import default + +if OSCheck.is_windows_family(): + from params_windows import * +else: + from params_linux import * + +host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) +nfsgateway_heapsize = config['configurations']['hadoop-env']['nfsgateway_heapsize'] +retryAble = default("/commandParams/command_retry_enabled", False) +script_https_protocol = Script.get_force_https_protocol_name() http://git-wip-us.apache.org/repos/asf/ambari/blob/8f051fc5/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py new file mode 100644 index 0000000..a77ec82 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_linux.py @@ -0,0 +1,527 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import status_params +import utils +import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. +import os +import re + +from ambari_commons.os_check import OSCheck + +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.stack_features import get_stack_feature_version +from resource_management.libraries.functions import format +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.expect import expect +from resource_management.libraries.functions import get_klist_path +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions.format_jvm_option import format_jvm_option +from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs +from resource_management.libraries.functions import is_empty + + +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +stack_name = status_params.stack_name +stack_root = Script.get_stack_root() +upgrade_direction = default("/commandParams/upgrade_direction", None) +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) +agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability'] +agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) + +# there is a stack upgrade which has not yet been finalized; it's currently suspended +upgrade_suspended = default("roleParams/upgrade_suspended", False) + +# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade +version = default("/commandParams/version", None) + +# The desired role is only available during a Non-Rolling Upgrade in HA. +# The server calculates which of the two NameNodes will be the active, and the other the standby since they +# are started using different commands. +desired_namenode_role = default("/commandParams/desired_namenode_role", None) + +# get the correct version to use for checking stack features +version_for_stack_feature_checks = get_stack_feature_version(config) + +stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks) +stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) + +security_enabled = config['configurations']['cluster-env']['security_enabled'] +hdfs_user = status_params.hdfs_user +root_user = "root" +hadoop_pid_dir_prefix = status_params.hadoop_pid_dir_prefix +namenode_pid_file = status_params.namenode_pid_file +zkfc_pid_file = status_params.zkfc_pid_file +datanode_pid_file = status_params.datanode_pid_file + +# Some datanode settings +dfs_dn_addr = default('/configurations/hdfs-site/dfs.datanode.address', None) +dfs_dn_http_addr = default('/configurations/hdfs-site/dfs.datanode.http.address', None) +dfs_dn_https_addr = default('/configurations/hdfs-site/dfs.datanode.https.address', None) +dfs_http_policy = default('/configurations/hdfs-site/dfs.http.policy', None) +dfs_dn_ipc_address = config['configurations']['hdfs-site']['dfs.datanode.ipc.address'] +secure_dn_ports_are_in_use = False + +hdfs_tmp_dir = default("/configurations/hadoop-env/hdfs_tmp_dir", "/tmp") +namenode_backup_dir = default("/configurations/hadoop-env/namenode_backup_dir", "/tmp/upgrades") + +# hadoop default parameters +mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*" +hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec") +hadoop_bin = stack_select.get_hadoop_dir("sbin") +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +hadoop_home = stack_select.get_hadoop_dir("home") +hadoop_secure_dn_user = hdfs_user +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +hadoop_conf_secure_dir = os.path.join(hadoop_conf_dir, "secure") +hadoop_lib_home = stack_select.get_hadoop_dir("lib") + +# hadoop parameters for stacks that support rolling_upgrade +if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): + mapreduce_libs_path = format("{stack_root}/current/hadoop-mapreduce-client/*") + + if not security_enabled: + hadoop_secure_dn_user = '""' + else: + dfs_dn_port = utils.get_port(dfs_dn_addr) + dfs_dn_http_port = utils.get_port(dfs_dn_http_addr) + dfs_dn_https_port = utils.get_port(dfs_dn_https_addr) + # We try to avoid inability to start datanode as a plain user due to usage of root-owned ports + if dfs_http_policy == "HTTPS_ONLY": + secure_dn_ports_are_in_use = utils.is_secure_port(dfs_dn_port) or utils.is_secure_port(dfs_dn_https_port) + elif dfs_http_policy == "HTTP_AND_HTTPS": + secure_dn_ports_are_in_use = utils.is_secure_port(dfs_dn_port) or utils.is_secure_port(dfs_dn_http_port) or utils.is_secure_port(dfs_dn_https_port) + else: # params.dfs_http_policy == "HTTP_ONLY" or not defined: + secure_dn_ports_are_in_use = utils.is_secure_port(dfs_dn_port) or utils.is_secure_port(dfs_dn_http_port) + if secure_dn_ports_are_in_use: + hadoop_secure_dn_user = hdfs_user + else: + hadoop_secure_dn_user = '""' + +ambari_libs_dir = "/var/lib/ambari-agent/lib" +limits_conf_dir = "/etc/security/limits.d" + +hdfs_user_nofile_limit = default("/configurations/hadoop-env/hdfs_user_nofile_limit", "128000") +hdfs_user_nproc_limit = default("/configurations/hadoop-env/hdfs_user_nproc_limit", "65536") + +create_lib_snappy_symlinks = check_stack_feature(StackFeature.SNAPPY, stack_version_formatted) +jsvc_path = "/usr/lib/bigtop-utils" + +execute_path = os.environ['PATH'] + os.pathsep + hadoop_bin_dir +ulimit_cmd = "ulimit -c unlimited ; " + +snappy_so = "libsnappy.so" +so_target_dir_x86 = format("{hadoop_lib_home}/native/Linux-i386-32") +so_target_dir_x64 = format("{hadoop_lib_home}/native/Linux-amd64-64") +so_target_x86 = format("{so_target_dir_x86}/{snappy_so}") +so_target_x64 = format("{so_target_dir_x64}/{snappy_so}") +so_src_dir_x86 = format("{hadoop_home}/lib") +so_src_dir_x64 = format("{hadoop_home}/lib64") +so_src_x86 = format("{so_src_dir_x86}/{snappy_so}") +so_src_x64 = format("{so_src_dir_x64}/{snappy_so}") + +#security params +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +falcon_user = config['configurations']['falcon-env']['falcon_user'] + +#exclude file +hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) +exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] +update_exclude_file_only = default("/commandParams/update_exclude_file_only",False) +command_phase = default("/commandParams/phase","") + +klist_path_local = get_klist_path(default('/configurations/kerberos-env/executable_search_paths', None)) +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +#hosts +hostname = config["hostname"] +rm_host = default("/clusterHostInfo/rm_host", []) +slave_hosts = default("/clusterHostInfo/slave_hosts", []) +oozie_servers = default("/clusterHostInfo/oozie_server", []) +hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", []) +hive_server_host = default("/clusterHostInfo/hive_server_host", []) +hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", []) +hs_host = default("/clusterHostInfo/hs_host", []) +jtnode_host = default("/clusterHostInfo/jtnode_host", []) +namenode_host = default("/clusterHostInfo/namenode_host", []) +nm_host = default("/clusterHostInfo/nm_host", []) +ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", []) +journalnode_hosts = default("/clusterHostInfo/journalnode_hosts", []) +zkfc_hosts = default("/clusterHostInfo/zkfc_hosts", []) +falcon_host = default("/clusterHostInfo/falcon_server_hosts", []) + +has_ganglia_server = not len(ganglia_server_hosts) == 0 +has_namenodes = not len(namenode_host) == 0 +has_jobtracker = not len(jtnode_host) == 0 +has_resourcemanager = not len(rm_host) == 0 +has_histroryserver = not len(hs_host) == 0 +has_hbase_masters = not len(hbase_master_hosts) == 0 +has_slaves = not len(slave_hosts) == 0 +has_oozie_server = not len(oozie_servers) == 0 +has_hcat_server_host = not len(hcat_server_hosts) == 0 +has_hive_server_host = not len(hive_server_host) == 0 +has_journalnode_hosts = not len(journalnode_hosts) == 0 +has_zkfc_hosts = not len(zkfc_hosts) == 0 +has_falcon_host = not len(falcon_host) == 0 + + +is_namenode_master = hostname in namenode_host +is_jtnode_master = hostname in jtnode_host +is_rmnode_master = hostname in rm_host +is_hsnode_master = hostname in hs_host +is_hbase_master = hostname in hbase_master_hosts +is_slave = hostname in slave_hosts + +if has_ganglia_server: + ganglia_server_host = ganglia_server_hosts[0] + +#users and groups +yarn_user = config['configurations']['yarn-env']['yarn_user'] +hbase_user = config['configurations']['hbase-env']['hbase_user'] +oozie_user = config['configurations']['oozie-env']['oozie_user'] +webhcat_user = config['configurations']['hive-env']['hcat_user'] +hcat_user = config['configurations']['hive-env']['hcat_user'] +hive_user = config['configurations']['hive-env']['hive_user'] +smoke_user = config['configurations']['cluster-env']['smokeuser'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +mapred_user = config['configurations']['mapred-env']['mapred_user'] +hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', None) + +user_group = config['configurations']['cluster-env']['user_group'] +root_group = "root" +proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group'] + +#hadoop params +hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] +hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger'] +nfs_file_dump_dir = config['configurations']['hdfs-site']['nfs.file.dump.dir'] + +dfs_domain_socket_path = config['configurations']['hdfs-site']['dfs.domain.socket.path'] +dfs_domain_socket_dir = os.path.dirname(dfs_domain_socket_path) + +jn_edits_dir = config['configurations']['hdfs-site']['dfs.journalnode.edits.dir'] + +dfs_name_dir = config['configurations']['hdfs-site']['dfs.namenode.name.dir'] + +hdfs_log_dir = format("{hdfs_log_dir_prefix}/{hdfs_user}") +namenode_dirs_created_stub_dir = hdfs_log_dir +namenode_dirs_stub_filename = "namenode_dirs_created" + +smoke_hdfs_user_dir = format("/user/{smoke_user}") +smoke_hdfs_user_mode = 0770 + +hdfs_namenode_format_disabled = default("/configurations/cluster-env/hdfs_namenode_format_disabled", False) +hdfs_namenode_formatted_mark_suffix = "/namenode-formatted/" +hdfs_namenode_bootstrapped_mark_suffix = "/namenode-bootstrapped/" +namenode_formatted_old_mark_dirs = ["/var/run/hadoop/hdfs/namenode-formatted", + format("{hadoop_pid_dir_prefix}/hdfs/namenode/formatted"), + "/var/lib/hdfs/namenode/formatted"] +dfs_name_dirs = dfs_name_dir.split(",") +namenode_formatted_mark_dirs = [] +namenode_bootstrapped_mark_dirs = [] +for dn_dir in dfs_name_dirs: + tmp_format_mark_dir = format("{dn_dir}{hdfs_namenode_formatted_mark_suffix}") + tmp_bootstrap_mark_dir = format("{dn_dir}{hdfs_namenode_bootstrapped_mark_suffix}") + namenode_formatted_mark_dirs.append(tmp_format_mark_dir) + namenode_bootstrapped_mark_dirs.append(tmp_bootstrap_mark_dir) + +# Use the namenode RPC address if configured, otherwise, fallback to the default file system +namenode_address = None +if 'dfs.namenode.rpc-address' in config['configurations']['hdfs-site']: + namenode_rpcaddress = config['configurations']['hdfs-site']['dfs.namenode.rpc-address'] + namenode_address = format("hdfs://{namenode_rpcaddress}") +else: + namenode_address = config['configurations']['core-site']['fs.defaultFS'] + +fs_checkpoint_dirs = default("/configurations/hdfs-site/dfs.namenode.checkpoint.dir", "").split(',') + +dfs_data_dirs = config['configurations']['hdfs-site']['dfs.datanode.data.dir'] + +data_dir_mount_file = "/var/lib/ambari-agent/data/datanode/dfs_data_dir_mount.hist" + +# HDFS High Availability properties +dfs_ha_enabled = False +dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.internal.nameservices', None) +if dfs_ha_nameservices is None: + dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.nameservices', None) +dfs_ha_namenode_ids = default(format("/configurations/hdfs-site/dfs.ha.namenodes.{dfs_ha_nameservices}"), None) +dfs_ha_automatic_failover_enabled = default("/configurations/hdfs-site/dfs.ha.automatic-failover.enabled", False) + +# hostname of the active HDFS HA Namenode (only used when HA is enabled) +dfs_ha_namenode_active = default("/configurations/hadoop-env/dfs_ha_initial_namenode_active", None) +# hostname of the standby HDFS HA Namenode (only used when HA is enabled) +dfs_ha_namenode_standby = default("/configurations/hadoop-env/dfs_ha_initial_namenode_standby", None) + +# Values for the current Host +namenode_id = None +namenode_rpc = None + +dfs_ha_namemodes_ids_list = [] +other_namenode_id = None + +if dfs_ha_namenode_ids: + dfs_ha_namemodes_ids_list = dfs_ha_namenode_ids.split(",") + dfs_ha_namenode_ids_array_len = len(dfs_ha_namemodes_ids_list) + if dfs_ha_namenode_ids_array_len > 1: + dfs_ha_enabled = True +if dfs_ha_enabled: + for nn_id in dfs_ha_namemodes_ids_list: + nn_host = config['configurations']['hdfs-site'][format('dfs.namenode.rpc-address.{dfs_ha_nameservices}.{nn_id}')] + if hostname in nn_host: + namenode_id = nn_id + namenode_rpc = nn_host + # With HA enabled namenode_address is recomputed + namenode_address = format('hdfs://{dfs_ha_nameservices}') + + # Calculate the namenode id of the other namenode. This is needed during RU to initiate an HA failover using ZKFC. + if namenode_id is not None and len(dfs_ha_namemodes_ids_list) == 2: + other_namenode_id = list(set(dfs_ha_namemodes_ids_list) - set([namenode_id]))[0] + + +if dfs_http_policy is not None and dfs_http_policy.upper() == "HTTPS_ONLY": + https_only = True + journalnode_address = default('/configurations/hdfs-site/dfs.journalnode.https-address', None) +else: + https_only = False + journalnode_address = default('/configurations/hdfs-site/dfs.journalnode.http-address', None) + +if journalnode_address: + journalnode_port = journalnode_address.split(":")[1] + + +if security_enabled: + dn_principal_name = config['configurations']['hdfs-site']['dfs.datanode.kerberos.principal'] + dn_keytab = config['configurations']['hdfs-site']['dfs.datanode.keytab.file'] + dn_principal_name = dn_principal_name.replace('_HOST',hostname.lower()) + + dn_kinit_cmd = format("{kinit_path_local} -kt {dn_keytab} {dn_principal_name};") + + nn_principal_name = config['configurations']['hdfs-site']['dfs.namenode.kerberos.principal'] + nn_keytab = config['configurations']['hdfs-site']['dfs.namenode.keytab.file'] + nn_principal_name = nn_principal_name.replace('_HOST',hostname.lower()) + + nn_kinit_cmd = format("{kinit_path_local} -kt {nn_keytab} {nn_principal_name};") + + jn_principal_name = default("/configurations/hdfs-site/dfs.journalnode.kerberos.principal", None) + if jn_principal_name: + jn_principal_name = jn_principal_name.replace('_HOST', hostname.lower()) + jn_keytab = default("/configurations/hdfs-site/dfs.journalnode.keytab.file", None) + hdfs_kinit_cmd = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};") +else: + dn_kinit_cmd = "" + nn_kinit_cmd = "" + hdfs_kinit_cmd = "" + +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] + +dfs_type = default("/commandParams/dfs_type", "") + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs, + immutable_paths = get_not_managed_resources(), + dfs_type = dfs_type +) + + +# The logic for LZO also exists in OOZIE's params.py +io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) +lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() +lzo_packages = ["lzo", "hadoop-lzo", "hadoop-lzo-native"] + +name_node_params = default("/commandParams/namenode", None) + +java_home = config['hostLevelParams']['java_home'] +java_version = expect("/hostLevelParams/java_version", int) + +hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] +namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize'] +namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize'] +namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize'] +namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m") +namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m") + +jtnode_opt_newsize = "200m" +jtnode_opt_maxnewsize = "200m" +jtnode_heapsize = "1024m" +ttnode_heapsize = "1024m" + +dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize'] +mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce") +mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce") + +# ranger host +ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) +has_ranger_admin = not len(ranger_admin_hosts) == 0 +xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] +ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] + +#ranger hdfs properties +policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] +if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'): + policymgr_mgr_url = policymgr_mgr_url.rstrip('/') +xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits') +xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') +xa_db_host = config['configurations']['admin-properties']['db_host'] +repo_name = str(config['clusterName']) + '_hadoop' + +hadoop_security_authentication = config['configurations']['core-site']['hadoop.security.authentication'] +hadoop_security_authorization = config['configurations']['core-site']['hadoop.security.authorization'] +fs_default_name = config['configurations']['core-site']['fs.defaultFS'] +hadoop_security_auth_to_local = config['configurations']['core-site']['hadoop.security.auth_to_local'] +hadoop_rpc_protection = config['configurations']['ranger-hdfs-plugin-properties']['hadoop.rpc.protection'] +common_name_for_certificate = config['configurations']['ranger-hdfs-plugin-properties']['common.name.for.certificate'] + +repo_config_username = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] + +if security_enabled: + sn_principal_name = default("/configurations/hdfs-site/dfs.secondary.namenode.kerberos.principal", "nn/[email protected]") + sn_principal_name = sn_principal_name.replace('_HOST',hostname.lower()) + +ranger_env = config['configurations']['ranger-env'] +ranger_plugin_properties = config['configurations']['ranger-hdfs-plugin-properties'] +policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_user'] + +#For curl command in ranger plugin to get db connector +jdk_location = config['hostLevelParams']['jdk_location'] +java_share_dir = '/usr/share/java' + +is_https_enabled = is_https_enabled_in_hdfs(config['configurations']['hdfs-site']['dfs.http.policy'], + config['configurations']['hdfs-site']['dfs.https.enable']) + +if has_ranger_admin: + enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes') + xa_audit_db_password = '' + if not is_empty(config['configurations']['admin-properties']['audit_db_password']) and stack_supports_ranger_audit_db: + xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) + repo_config_password = unicode(config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) + xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() + previous_jdbc_jar_name = None + + if stack_supports_ranger_audit_db: + + if xa_audit_db_flavor == 'mysql': + jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) + audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') + jdbc_driver = "com.mysql.jdbc.Driver" + elif xa_audit_db_flavor == 'oracle': + jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) + colon_count = xa_db_host.count(':') + if colon_count == 2 or colon_count == 0: + audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}') + else: + audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}') + jdbc_driver = "oracle.jdbc.OracleDriver" + elif xa_audit_db_flavor == 'postgres': + jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) + audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}') + jdbc_driver = "org.postgresql.Driver" + elif xa_audit_db_flavor == 'mssql': + jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) + audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}') + jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" + elif xa_audit_db_flavor == 'sqla': + jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) + audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') + jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" + + downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + driver_curl_target = format("{hadoop_lib_home}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + previous_jdbc_jar = format("{hadoop_lib_home}/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None + + sql_connector_jar = '' + + hdfs_ranger_plugin_config = { + 'username': repo_config_username, + 'password': repo_config_password, + 'hadoop.security.authentication': hadoop_security_authentication, + 'hadoop.security.authorization': hadoop_security_authorization, + 'fs.default.name': fs_default_name, + 'hadoop.security.auth_to_local': hadoop_security_auth_to_local, + 'hadoop.rpc.protection': hadoop_rpc_protection, + 'commonNameForCertificate': common_name_for_certificate, + 'dfs.datanode.kerberos.principal': dn_principal_name if security_enabled else '', + 'dfs.namenode.kerberos.principal': nn_principal_name if security_enabled else '', + 'dfs.secondary.namenode.kerberos.principal': sn_principal_name if security_enabled else '' + } + + hdfs_ranger_plugin_repo = { + 'isActive': 'true', + 'config': json.dumps(hdfs_ranger_plugin_config), + 'description': 'hdfs repo', + 'name': repo_name, + 'repositoryType': 'hdfs', + 'assetType': '1' + } + if stack_supports_ranger_kerberos and security_enabled: + hdfs_ranger_plugin_config['policy.download.auth.users'] = hdfs_user + hdfs_ranger_plugin_config['tag.download.auth.users'] = hdfs_user + + if stack_supports_ranger_kerberos: + hdfs_ranger_plugin_config['ambari.service.check.user'] = policy_user + + hdfs_ranger_plugin_repo = { + 'isEnabled': 'true', + 'configs': hdfs_ranger_plugin_config, + 'description': 'hdfs repo', + 'name': repo_name, + 'type': 'hdfs' + } + + xa_audit_db_is_enabled = False + ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] + if xml_configurations_supported and stack_supports_ranger_audit_db: + xa_audit_db_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.db'] + xa_audit_hdfs_is_enabled = config['configurations']['ranger-hdfs-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None + ssl_keystore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None + ssl_truststore_password = unicode(config['configurations']['ranger-hdfs-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None + + #For SQLA explicitly disable audit to DB for Ranger + if xa_audit_db_flavor == 'sqla': + xa_audit_db_is_enabled = False http://git-wip-us.apache.org/repos/asf/ambari/blob/8f051fc5/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_windows.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_windows.py new file mode 100644 index 0000000..da00985 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/params_windows.py @@ -0,0 +1,76 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os + +#Used in subsequent imports from params +from install_params import exclude_packages +from status_params import * + +config = Script.get_config() +hadoop_conf_dir = None +hbase_conf_dir = None +hadoop_home = None +try: + hadoop_conf_dir = os.environ["HADOOP_CONF_DIR"] + hbase_conf_dir = os.environ["HBASE_CONF_DIR"] + hadoop_home = os.environ["HADOOP_HOME"] +except: + pass +#directories & files +dfs_name_dir = config['configurations']['hdfs-site']['dfs.namenode.name.dir'] +fs_checkpoint_dir = config['configurations']['hdfs-site']['dfs.namenode.checkpoint.dir'] +dfs_data_dir = config['configurations']['hdfs-site']['dfs.datanode.data.dir'] +#decomission +hdfs_exclude_file = default("/clusterHostInfo/decom_dn_hosts", []) +exclude_file_path = config['configurations']['hdfs-site']['dfs.hosts.exclude'] +# HDFS High Availability properties +dfs_ha_enabled = False +dfs_ha_nameservices = default("/configurations/hdfs-site/dfs.internal.nameservices", None) +dfs_ha_namenode_ids = default(format("/configurations/hdfs-site/dfs.ha.namenodes.{dfs_ha_nameservices}"), None) + +namenode_id = None +namenode_rpc = None +hostname = config["hostname"] +if dfs_ha_namenode_ids: + dfs_ha_namemodes_ids_list = dfs_ha_namenode_ids.split(",") + dfs_ha_namenode_ids_array_len = len(dfs_ha_namemodes_ids_list) + if dfs_ha_namenode_ids_array_len > 1: + dfs_ha_enabled = True +if dfs_ha_enabled: + for nn_id in dfs_ha_namemodes_ids_list: + nn_host = config['configurations']['hdfs-site'][format('dfs.namenode.rpc-address.{dfs_ha_nameservices}.{nn_id}')] + if hostname in nn_host: + namenode_id = nn_id + namenode_rpc = nn_host + +hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"] +hdfs_user = hadoop_user + +grep_exe = "findstr" + +name_node_params = default("/commandParams/namenode", None) + +service_map = { + "datanode" : datanode_win_service_name, + "journalnode" : journalnode_win_service_name, + "namenode" : namenode_win_service_name, + "secondarynamenode" : snamenode_win_service_name, + "zkfc_slave": zkfc_win_service_name +} http://git-wip-us.apache.org/repos/asf/ambari/blob/8f051fc5/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/service_check.py new file mode 100644 index 0000000..064fbfc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/service_check.py @@ -0,0 +1,153 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from resource_management.core.shell import as_user +from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons import OSConst +from resource_management.libraries.functions.curl_krb_request import curl_krb_request +from resource_management.core.logger import Logger + +class HdfsServiceCheck(Script): + pass + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class HdfsServiceCheckDefault(HdfsServiceCheck): + def service_check(self, env): + import params + + env.set_params(params) + unique = functions.get_unique_id_and_date() + dir = params.hdfs_tmp_dir + tmp_file = format("{dir}/{unique}") + + """ + Ignore checking safemode, because this command is unable to get safemode state + when 1 namenode is down in an HA setup (see more in HDFS-8277). Directly + test HDFS availability by file system operations is consistent in both HA and + non-HA environment. + """ + # safemode_command = format("dfsadmin -fs {namenode_address} -safemode get | grep OFF") + + if params.security_enabled: + Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"), + user=params.hdfs_user + ) + #ExecuteHadoop(safemode_command, + # user=params.hdfs_user, + # logoutput=True, + # conf_dir=params.hadoop_conf_dir, + # try_sleep=3, + # tries=20, + # bin_dir=params.hadoop_bin_dir + #) + params.HdfsResource(dir, + type="directory", + action="create_on_execute", + mode=0777 + ) + params.HdfsResource(tmp_file, + type="file", + action="delete_on_execute", + ) + + params.HdfsResource(tmp_file, + type="file", + source="/etc/passwd", + action="create_on_execute" + ) + params.HdfsResource(None, action="execute") + + if params.has_journalnode_hosts: + if params.security_enabled: + for host in params.journalnode_hosts: + if params.https_only: + uri = format("https://{host}:{journalnode_port}";) + else: + uri = format("http://{host}:{journalnode_port}";) + response, errmsg, time_millis = curl_krb_request(params.tmp_dir, params.smoke_user_keytab, + params.smokeuser_principal, uri, "jn_service_check", + params.kinit_path_local, False, None, params.smoke_user) + if not response: + Logger.error("Cannot access WEB UI on: {0}. Error : {1}", uri, errmsg) + return 1 + else: + journalnode_port = params.journalnode_port + checkWebUIFileName = "checkWebUI.py" + checkWebUIFilePath = format("{tmp_dir}/{checkWebUIFileName}") + comma_sep_jn_hosts = ",".join(params.journalnode_hosts) + + checkWebUICmd = format("ambari-python-wrap {checkWebUIFilePath} -m {comma_sep_jn_hosts} -p {journalnode_port} -s {https_only} -o {script_https_protocol}") + File(checkWebUIFilePath, + content=StaticFile(checkWebUIFileName), + mode=0775) + + Execute(checkWebUICmd, + logoutput=True, + try_sleep=3, + tries=5, + user=params.smoke_user + ) + + if params.is_namenode_master: + if params.has_zkfc_hosts: + pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}") + pid_file = format("{pid_dir}/hadoop-{hdfs_user}-zkfc.pid") + check_zkfc_process_cmd = as_user(format( + "ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.hdfs_user) + Execute(check_zkfc_process_cmd, + logoutput=True, + try_sleep=3, + tries=5 + ) + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class HdfsServiceCheckWindows(HdfsServiceCheck): + def service_check(self, env): + import params + env.set_params(params) + + unique = functions.get_unique_id_and_date() + + #Hadoop uses POSIX-style paths, separator is always / + dir = params.hdfs_tmp_dir + tmp_file = dir + '/' + unique + + #commands for execution + hadoop_cmd = "cmd /C %s" % (os.path.join(params.hadoop_home, "bin", "hadoop.cmd")) + create_dir_cmd = "%s fs -mkdir %s" % (hadoop_cmd, dir) + own_dir = "%s fs -chmod 777 %s" % (hadoop_cmd, dir) + test_dir_exists = "%s fs -test -e %s" % (hadoop_cmd, dir) + cleanup_cmd = "%s fs -rm %s" % (hadoop_cmd, tmp_file) + create_file_cmd = "%s fs -put %s %s" % (hadoop_cmd, os.path.join(params.hadoop_conf_dir, "core-site.xml"), tmp_file) + test_cmd = "%s fs -test -e %s" % (hadoop_cmd, tmp_file) + + hdfs_cmd = "cmd /C %s" % (os.path.join(params.hadoop_home, "bin", "hdfs.cmd")) + safemode_command = "%s dfsadmin -safemode get | %s OFF" % (hdfs_cmd, params.grep_exe) + + Execute(safemode_command, logoutput=True, try_sleep=3, tries=20) + Execute(create_dir_cmd, user=params.hdfs_user,logoutput=True, ignore_failures=True) + Execute(own_dir, user=params.hdfs_user,logoutput=True) + Execute(test_dir_exists, user=params.hdfs_user,logoutput=True) + Execute(create_file_cmd, user=params.hdfs_user,logoutput=True) + Execute(test_cmd, user=params.hdfs_user,logoutput=True) + Execute(cleanup_cmd, user=params.hdfs_user,logoutput=True) + +if __name__ == "__main__": + HdfsServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/8f051fc5/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/setup_ranger_hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/setup_ranger_hdfs.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/setup_ranger_hdfs.py new file mode 100644 index 0000000..e3aff9d --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/setup_ranger_hdfs.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.constants import Direction +from resource_management.libraries.functions.format import format + + +def setup_ranger_hdfs(upgrade_type=None): + import params + + if params.has_ranger_admin: + + + stack_version = None + + if upgrade_type is not None: + stack_version = params.version + + if params.retryAble: + Logger.info("HDFS: Setup ranger: command retry enables thus retrying if ranger admin is down !") + else: + Logger.info("HDFS: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + + + if params.xml_configurations_supported: + from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin + api_version=None + if params.stack_supports_ranger_kerberos: + api_version='v2' + setup_ranger_plugin('hadoop-client', 'hdfs', params.previous_jdbc_jar, + params.downloaded_custom_connector, params.driver_curl_source, + params.driver_curl_target, params.java_home, + params.repo_name, params.hdfs_ranger_plugin_repo, + params.ranger_env, params.ranger_plugin_properties, + params.policy_user, params.policymgr_mgr_url, + params.enable_ranger_hdfs, conf_dict=params.hadoop_conf_dir, + component_user=params.hdfs_user, component_group=params.user_group, cache_service_list=['hdfs'], + plugin_audit_properties=params.config['configurations']['ranger-hdfs-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-hdfs-audit'], + plugin_security_properties=params.config['configurations']['ranger-hdfs-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-hdfs-security'], + plugin_policymgr_ssl_properties=params.config['configurations']['ranger-hdfs-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-hdfs-policymgr-ssl'], + component_list=['hadoop-client'], audit_db_is_enabled=params.xa_audit_db_is_enabled, + credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, + ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, + api_version=api_version ,stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble, + is_security_enabled = params.security_enabled, + is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos, + component_user_principal=params.nn_principal_name if params.security_enabled else None, + component_user_keytab=params.nn_keytab if params.security_enabled else None) + else: + from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin + + setup_ranger_plugin('hadoop-client', 'hdfs', params.previous_jdbc_jar, + params.downloaded_custom_connector, params.driver_curl_source, + params.driver_curl_target, params.java_home, + params.repo_name, params.hdfs_ranger_plugin_repo, + params.ranger_env, params.ranger_plugin_properties, + params.policy_user, params.policymgr_mgr_url, + params.enable_ranger_hdfs, conf_dict=params.hadoop_conf_dir, + component_user=params.hdfs_user, component_group=params.user_group, cache_service_list=['hdfs'], + plugin_audit_properties=params.config['configurations']['ranger-hdfs-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-hdfs-audit'], + plugin_security_properties=params.config['configurations']['ranger-hdfs-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-hdfs-security'], + plugin_policymgr_ssl_properties=params.config['configurations']['ranger-hdfs-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-hdfs-policymgr-ssl'], + component_list=['hadoop-client'], audit_db_is_enabled=params.xa_audit_db_is_enabled, + credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, + ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, + stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble) + + if stack_version and params.upgrade_direction == Direction.UPGRADE: + # when upgrading to stack remove_ranger_hdfs_plugin_env, this env file must be removed + if check_stack_feature(StackFeature.REMOVE_RANGER_HDFS_PLUGIN_ENV, stack_version): + source_file = os.path.join(params.hadoop_conf_dir, 'set-hdfs-plugin-env.sh') + target_file = source_file + ".bak" + Execute(("mv", source_file, target_file), sudo=True, only_if=format("test -f {source_file}")) + else: + Logger.info('Ranger admin not installed') + +def create_ranger_audit_hdfs_directories(): + import params + + if params.has_ranger_admin: + if params.xml_configurations_supported and params.enable_ranger_hdfs and params.xa_audit_hdfs_is_enabled: + params.HdfsResource("/ranger/audit", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0755, + recursive_chmod=True, + ) + params.HdfsResource("/ranger/audit/hdfs", + type="directory", + action="create_on_execute", + owner=params.hdfs_user, + group=params.hdfs_user, + mode=0700, + recursive_chmod=True, + ) + params.HdfsResource(None, action="execute") + else: + Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/8f051fc5/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/snamenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/snamenode.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/snamenode.py new file mode 100644 index 0000000..30eee07 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/snamenode.py @@ -0,0 +1,152 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.security_commons import build_expectations, \ + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML + +from hdfs_snamenode import snamenode +from hdfs import hdfs +from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons import OSConst + +from resource_management.core.logger import Logger + +class SNameNode(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + hdfs("secondarynamenode") + snamenode(action="configure") + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + snamenode(action="start") + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + snamenode(action="stop") + + def status(self, env): + import status_params + env.set_params(status_params) + snamenode(action="status") + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class SNameNodeDefault(SNameNode): + + def get_component_name(self): + return "hadoop-hdfs-secondarynamenode" + + def pre_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing Stack Upgrade pre-restart") + import params + env.set_params(params) + + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + conf_select.select(params.stack_name, "hadoop", params.version) + stack_select.select("hadoop-hdfs-secondarynamenode", params.version) + + def security_status(self, env): + import status_params + + env.set_params(status_params) + props_value_check = {"hadoop.security.authentication": "kerberos", + "hadoop.security.authorization": "true"} + props_empty_check = ["hadoop.security.auth_to_local"] + props_read_check = None + core_site_expectations = build_expectations('core-site', props_value_check, props_empty_check, + props_read_check) + props_value_check = None + props_empty_check = ['dfs.secondary.namenode.kerberos.internal.spnego.principal', + 'dfs.secondary.namenode.keytab.file', + 'dfs.secondary.namenode.kerberos.principal'] + props_read_check = ['dfs.secondary.namenode.keytab.file'] + hdfs_site_expectations = build_expectations('hdfs-site', props_value_check, props_empty_check, + props_read_check) + + hdfs_expectations = {} + hdfs_expectations.update(core_site_expectations) + hdfs_expectations.update(hdfs_site_expectations) + + security_params = get_params_from_filesystem(status_params.hadoop_conf_dir, + {'core-site.xml': FILE_TYPE_XML, + 'hdfs-site.xml': FILE_TYPE_XML}) + + if 'core-site' in security_params and 'hadoop.security.authentication' in security_params['core-site'] and \ + security_params['core-site']['hadoop.security.authentication'].lower() == 'kerberos': + result_issues = validate_security_config_properties(security_params, hdfs_expectations) + if not result_issues: # If all validations passed successfully + try: + # Double check the dict before calling execute + if ('hdfs-site' not in security_params or + 'dfs.secondary.namenode.keytab.file' not in security_params['hdfs-site'] or + 'dfs.secondary.namenode.kerberos.principal' not in security_params['hdfs-site']): + self.put_structured_out({"securityState": "UNSECURED"}) + self.put_structured_out( + {"securityIssuesFound": "Keytab file or principal are not set property."}) + return + + cached_kinit_executor(status_params.kinit_path_local, + status_params.hdfs_user, + security_params['hdfs-site']['dfs.secondary.namenode.keytab.file'], + security_params['hdfs-site'][ + 'dfs.secondary.namenode.kerberos.principal'], + status_params.hostname, + status_params.tmp_dir) + self.put_structured_out({"securityState": "SECURED_KERBEROS"}) + except Exception as e: + self.put_structured_out({"securityState": "ERROR"}) + self.put_structured_out({"securityStateErrorInfo": str(e)}) + else: + issues = [] + for cf in result_issues: + issues.append("Configuration file %s did not pass the validation. Reason: %s" % (cf, result_issues[cf])) + self.put_structured_out({"securityIssuesFound": ". ".join(issues)}) + self.put_structured_out({"securityState": "UNSECURED"}) + else: + self.put_structured_out({"securityState": "UNSECURED"}) + + def get_log_folder(self): + import params + return params.hdfs_log_dir + + def get_user(self): + import params + return params.hdfs_user + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class SNameNodeWindows(SNameNode): + pass + +if __name__ == "__main__": + SNameNode().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/8f051fc5/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/status_params.py new file mode 100644 index 0000000..153f9a6 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HDFS/package/scripts/status_params.py @@ -0,0 +1,58 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from ambari_commons import OSCheck + +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.script.script import Script + +config = Script.get_config() + +if OSCheck.is_windows_family(): + namenode_win_service_name = "namenode" + datanode_win_service_name = "datanode" + snamenode_win_service_name = "secondarynamenode" + journalnode_win_service_name = "journalnode" + zkfc_win_service_name = "zkfc" +else: + hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix'] + hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] + hadoop_pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}") + datanode_pid_file = format("{hadoop_pid_dir}/hadoop-{hdfs_user}-datanode.pid") + namenode_pid_file = format("{hadoop_pid_dir}/hadoop-{hdfs_user}-namenode.pid") + snamenode_pid_file = format("{hadoop_pid_dir}/hadoop-{hdfs_user}-secondarynamenode.pid") + journalnode_pid_file = format("{hadoop_pid_dir}/hadoop-{hdfs_user}-journalnode.pid") + zkfc_pid_file = format("{hadoop_pid_dir}/hadoop-{hdfs_user}-zkfc.pid") + nfsgateway_pid_file = format("{hadoop_pid_dir_prefix}/root/hadoop_privileged_nfs3.pid") + + # Security related/required params + hostname = config['hostname'] + security_enabled = config['configurations']['cluster-env']['security_enabled'] + hdfs_user_principal = config['configurations']['hadoop-env']['hdfs_principal_name'] + hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] + + hadoop_conf_dir = conf_select.get_hadoop_conf_dir() + + kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) + tmp_dir = Script.get_tmp_dir() + +stack_name = default("/hostLevelParams/stack_name", None) \ No newline at end of file
