Repository: ambari Updated Branches: refs/heads/trunk b3f7d9e42 -> 4b588a923
http://git-wip-us.apache.org/repos/asf/ambari/blob/4b588a92/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/setup_livy.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/setup_livy.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/setup_livy.py deleted file mode 100644 index adaca87..0000000 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/setup_livy.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import os -from resource_management import Directory, File, PropertiesFile, InlineTemplate, format - - -def setup_livy(env, type, upgrade_type = None, action = None): - import params - - Directory([params.livy_pid_dir, params.livy_log_dir], - owner=params.livy_user, - group=params.user_group, - mode=0775, - create_parents = True - ) - if type == 'server' and action == 'config': - params.HdfsResource(params.livy_hdfs_user_dir, - type="directory", - action="create_on_execute", - owner=params.livy_user, - mode=0775 - ) - params.HdfsResource(None, action="execute") - - params.HdfsResource(params.livy_recovery_dir, - type="directory", - action="create_on_execute", - owner=params.livy_user, - mode=0700 - ) - params.HdfsResource(None, action="execute") - - # create livy-env.sh in etc/conf dir - File(os.path.join(params.livy_conf, 'livy-env.sh'), - owner=params.livy_user, - group=params.livy_group, - content=InlineTemplate(params.livy_env_sh), - mode=0644, - ) - - # create livy.conf in etc/conf dir - PropertiesFile(format("{livy_conf}/livy.conf"), - properties = params.config['configurations']['livy-conf'], - key_value_delimiter = " ", - owner=params.livy_user, - group=params.livy_group, - ) - - # create log4j.properties in etc/conf dir - File(os.path.join(params.livy_conf, 'log4j.properties'), - owner=params.livy_user, - group=params.livy_group, - content=params.livy_log4j_properties, - mode=0644, - ) - - # create spark-blacklist.properties in etc/conf dir - File(os.path.join(params.livy_conf, 'spark-blacklist.conf'), - owner=params.livy_user, - group=params.livy_group, - content=params.livy_spark_blacklist_properties, - mode=0644, - ) - - Directory(params.livy_logs_dir, - owner=params.livy_user, - group=params.livy_group, - mode=0755, - ) - http://git-wip-us.apache.org/repos/asf/ambari/blob/4b588a92/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/setup_spark.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/setup_spark.py deleted file mode 100644 index 9329ce0..0000000 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/setup_spark.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -import fileinput -import shutil -import os - -from resource_management.core.exceptions import ComponentIsNotRunning -from resource_management.core.logger import Logger -from resource_management.core import shell -from resource_management.core.source import InlineTemplate -from resource_management.core.resources.system import Directory, File -from resource_management.libraries.resources.properties_file import PropertiesFile -from resource_management.libraries.functions.version import format_stack_version -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.constants import StackFeature -from resource_management.libraries.functions.format import format -from resource_management.libraries.resources.xml_config import XmlConfig - -def setup_spark(env, type, upgrade_type = None, action = None): - import params - - Directory([params.spark_pid_dir, params.spark_log_dir], - owner=params.spark_user, - group=params.user_group, - mode=0775, - create_parents = True - ) - if type == 'server' and action == 'config': - params.HdfsResource(params.spark_hdfs_user_dir, - type="directory", - action="create_on_execute", - owner=params.spark_user, - mode=0775 - ) - params.HdfsResource(None, action="execute") - - PropertiesFile(format("{spark_conf}/spark-defaults.conf"), - properties = params.config['configurations']['spark-defaults'], - key_value_delimiter = " ", - owner=params.spark_user, - group=params.spark_group, - mode=0644 - ) - - # create spark-env.sh in etc/conf dir - File(os.path.join(params.spark_conf, 'spark-env.sh'), - owner=params.spark_user, - group=params.spark_group, - content=InlineTemplate(params.spark_env_sh), - mode=0644, - ) - - #create log4j.properties in etc/conf dir - File(os.path.join(params.spark_conf, 'log4j.properties'), - owner=params.spark_user, - group=params.spark_group, - content=params.spark_log4j_properties, - mode=0644, - ) - - #create metrics.properties in etc/conf dir - File(os.path.join(params.spark_conf, 'metrics.properties'), - owner=params.spark_user, - group=params.spark_group, - content=InlineTemplate(params.spark_metrics_properties), - mode=0644 - ) - - if params.is_hive_installed: - XmlConfig("hive-site.xml", - conf_dir=params.spark_conf, - configurations=params.spark_hive_properties, - owner=params.spark_user, - group=params.spark_group, - mode=0644) - - if params.has_spark_thriftserver: - PropertiesFile(params.spark_thrift_server_conf_file, - properties = params.config['configurations']['spark-thrift-sparkconf'], - owner = params.hive_user, - group = params.user_group, - key_value_delimiter = " ", - mode=0644 - ) - - effective_version = params.version if upgrade_type is not None else params.stack_version_formatted - if effective_version: - effective_version = format_stack_version(effective_version) - - if params.spark_thrift_fairscheduler_content and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): - # create spark-thrift-fairscheduler.xml - File(os.path.join(params.spark_conf,"spark-thrift-fairscheduler.xml"), - owner=params.spark_user, - group=params.spark_group, - mode=0755, - content=InlineTemplate(params.spark_thrift_fairscheduler_content) - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/4b588a92/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_client.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_client.py deleted file mode 100644 index 3acde4e..0000000 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_client.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -from resource_management.libraries.script.script import Script -from resource_management.libraries.functions import conf_select, stack_select -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.constants import StackFeature -from resource_management.core.exceptions import ClientComponentHasNoStatus -from resource_management.core.logger import Logger -from resource_management.core import shell -from setup_spark import setup_spark - - -class SparkClient(Script): - def install(self, env): - self.install_packages(env) - self.configure(env) - - def configure(self, env, upgrade_type=None, config_dir=None): - import params - env.set_params(params) - - setup_spark(env, 'client', upgrade_type=upgrade_type, action = 'config') - - def status(self, env): - raise ClientComponentHasNoStatus() - - def get_component_name(self): - # TODO, change to "spark" after RPM switches the name - return "spark2-client" - - def pre_upgrade_restart(self, env, upgrade_type=None): - import params - - env.set_params(params) - if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): - Logger.info("Executing Spark Client Stack Upgrade pre-restart") - # TODO, change to "spark" after RPM switches the name - conf_select.select(params.stack_name, "spark2", params.version) - stack_select.select("spark2-client", params.version) - -if __name__ == "__main__": - SparkClient().execute() - http://git-wip-us.apache.org/repos/asf/ambari/blob/4b588a92/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_service.py deleted file mode 100644 index 536d798..0000000 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_service.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python - -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' -import socket -import tarfile -import os -from contextlib import closing - -from resource_management.libraries.script.script import Script -from resource_management.libraries.resources.hdfs_resource import HdfsResource -from resource_management.libraries.functions.copy_tarball import copy_to_hdfs, get_tarball_paths -from resource_management.libraries.functions import format -from resource_management.core.resources.system import File, Execute -from resource_management.libraries.functions.version import format_stack_version -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.constants import StackFeature -from resource_management.libraries.functions.show_logs import show_logs - - -def make_tarfile(output_filename, source_dir): - try: - os.remove(output_filename) - except OSError: - pass - parent_dir=os.path.dirname(output_filename) - if not os.path.exists(parent_dir): - os.makedirs(parent_dir) - os.chmod(parent_dir, 0711) - with closing(tarfile.open(output_filename, "w:gz")) as tar: - for file in os.listdir(source_dir): - tar.add(os.path.join(source_dir,file),arcname=file) - os.chmod(output_filename, 0644) - - -def spark_service(name, upgrade_type=None, action=None): - import params - - if action == 'start': - - effective_version = params.version if upgrade_type is not None else params.stack_version_formatted - if effective_version: - effective_version = format_stack_version(effective_version) - - if name == 'jobhistoryserver' and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): - # TODO, change to "spark" after RPM switches the name - # create & copy spark2-hdp-yarn-archive.tar.gz to hdfs - if not params.sysprep_skip_copy_tarballs_hdfs: - source_dir=params.spark_home+"/jars" - tmp_archive_file=get_tarball_paths("spark2")[1] - make_tarfile(tmp_archive_file, source_dir) - copy_to_hdfs("spark2", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs, replace_existing_files=True) - # create spark history directory - params.HdfsResource(params.spark_history_dir, - type="directory", - action="create_on_execute", - owner=params.spark_user, - group=params.user_group, - mode=0777, - recursive_chmod=True - ) - params.HdfsResource(None, action="execute") - - if params.security_enabled: - spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ") - Execute(spark_kinit_cmd, user=params.spark_user) - - # Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not - # need to copy the tarball, otherwise, copy it. - if params.stack_version_formatted and check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.stack_version_formatted): - resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs) - if resource_created: - params.HdfsResource(None, action="execute") - - if name == 'jobhistoryserver': - historyserver_no_op_test = format( - 'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1') - try: - Execute(format('{spark_history_server_start}'), - user=params.spark_user, - environment={'JAVA_HOME': params.java_home}, - not_if=historyserver_no_op_test) - except: - show_logs(params.spark_log_dir, user=params.spark_user) - raise - - elif name == 'sparkthriftserver': - if params.security_enabled: - hive_principal = params.hive_kerberos_principal - hive_kinit_cmd = format("{kinit_path_local} -kt {hive_kerberos_keytab} {hive_principal}; ") - Execute(hive_kinit_cmd, user=params.hive_user) - - thriftserver_no_op_test = format( - 'ls {spark_thrift_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_thrift_server_pid_file}` >/dev/null 2>&1') - try: - Execute(format('{spark_thrift_server_start} --properties-file {spark_thrift_server_conf_file} {spark_thrift_cmd_opts_properties}'), - user=params.hive_user, - environment={'JAVA_HOME': params.java_home}, - not_if=thriftserver_no_op_test - ) - except: - show_logs(params.spark_log_dir, user=params.hive_user) - raise - elif action == 'stop': - if name == 'jobhistoryserver': - try: - Execute(format('{spark_history_server_stop}'), - user=params.spark_user, - environment={'JAVA_HOME': params.java_home} - ) - except: - show_logs(params.spark_log_dir, user=params.spark_user) - raise - File(params.spark_history_server_pid_file, - action="delete" - ) - - elif name == 'sparkthriftserver': - try: - Execute(format('{spark_thrift_server_stop}'), - user=params.hive_user, - environment={'JAVA_HOME': params.java_home} - ) - except: - show_logs(params.spark_log_dir, user=params.hive_user) - raise - File(params.spark_thrift_server_pid_file, - action="delete" - ) - - http://git-wip-us.apache.org/repos/asf/ambari/blob/4b588a92/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_thrift_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_thrift_server.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_thrift_server.py deleted file mode 100644 index 8953b35..0000000 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/spark_thrift_server.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -import os - -from resource_management.libraries.script.script import Script -from resource_management.libraries.functions import conf_select, stack_select -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.constants import StackFeature -from resource_management.libraries.functions.check_process_status import check_process_status -from resource_management.core.logger import Logger -from resource_management.core import shell -from setup_spark import setup_spark -from spark_service import spark_service - - -class SparkThriftServer(Script): - - def install(self, env): - import params - env.set_params(params) - - self.install_packages(env) - - def configure(self, env, upgrade_type=None, config_dir=None): - import params - env.set_params(params) - setup_spark(env, 'server', upgrade_type = upgrade_type, action = 'config') - - def start(self, env, upgrade_type=None): - import params - env.set_params(params) - - self.configure(env) - spark_service('sparkthriftserver', upgrade_type=upgrade_type, action='start') - - def stop(self, env, upgrade_type=None): - import params - env.set_params(params) - spark_service('sparkthriftserver', upgrade_type=upgrade_type, action='stop') - - def status(self, env): - import status_params - env.set_params(status_params) - check_process_status(status_params.spark_thrift_server_pid_file) - - def get_component_name(self): - # TODO, change to "spark" after RPM switches the name - return "spark2-thriftserver" - - def pre_upgrade_restart(self, env, upgrade_type=None): - import params - - env.set_params(params) - Logger.info("Executing Spark Thrift Server Stack Upgrade pre-restart") - # TODO, change to "spark" after RPM switches the name - conf_select.select(params.stack_name, "spark2", params.version) - stack_select.select("spark2-thriftserver", params.version) - - def get_log_folder(self): - import params - return params.spark_log_dir - - def get_user(self): - import params - return params.hive_user - - def get_pid_files(self): - import status_params - return [status_params.spark_thrift_server_pid_file] - -if __name__ == "__main__": - SparkThriftServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/4b588a92/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/status_params.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/status_params.py deleted file mode 100644 index 07dcc47..0000000 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/scripts/status_params.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management.libraries.functions.format import format -from resource_management.libraries.script.script import Script -from resource_management.libraries.functions.default import default - -config = Script.get_config() - -spark_user = config['configurations']['spark-env']['spark_user'] -spark_group = config['configurations']['spark-env']['spark_group'] -user_group = config['configurations']['cluster-env']['user_group'] - -if 'hive-env' in config['configurations']: - hive_user = config['configurations']['hive-env']['hive_user'] -else: - hive_user = "hive" - -spark_pid_dir = config['configurations']['spark-env']['spark_pid_dir'] -spark_history_server_pid_file = format("{spark_pid_dir}/spark-{spark_user}-org.apache.spark.deploy.history.HistoryServer-1.pid") -spark_thrift_server_pid_file = format("{spark_pid_dir}/spark-{hive_user}-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid") -stack_name = default("/hostLevelParams/stack_name", None) - -if "livy-env" in config['configurations']: - livy_user = config['configurations']['livy-env']['livy_user'] - livy_group = config['configurations']['livy-env']['livy_group'] - livy_pid_dir = config['configurations']['livy-env']['livy_pid_dir'] - livy_server_pid_file = format("{livy_pid_dir}/livy-{livy_user}-server.pid") \ No newline at end of file
